perf(runtime): static use digests, process-level use prefetch, state-dir lock
A cell that uses notebooks by literal path is keyed by what each of them is, and the kernel caches such cells; the use calls a cell is about to make are started together in worker processes (SCA_JOBS), when that is worth the seconds a worker takes to start; a notebook's state directory is locked between processes. Desktop-arm benchmark: cold 80.5s -> 34.6s. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -23,6 +23,20 @@ user-facing notes live in [`docs/updates/`](docs/updates/) (English and
|
||||
cell, changed (marimo keyed any object it cannot hash by the code of the cell that made it).
|
||||
- Editor: a slider drag is sent when it pauses (250 ms) or ends, not step by step; clicks and
|
||||
number-box commits still go at once.
|
||||
- Cache: a cell that runs a notebook with `use("child.py")` (a literal path) is keyed by what
|
||||
that child is (its text, the modules it imports, the notebooks it uses in turn) instead of
|
||||
by everything a run recorded: editing one child redoes the cell that uses it and not those
|
||||
that use its siblings, and the editor kernel caches such cells too (it ran them live
|
||||
before). A path computed at run time still depends on what a run recorded.
|
||||
- Runtime: the `use` calls a cell is about to make (literal path; arguments made of values it
|
||||
reads, `float(...)`, `notebook_id()`) are started together in worker processes, which fill
|
||||
the notebooks' caches; the cell then finds them built. OpenCASCADE keeps the interpreter
|
||||
lock, so these are processes, not threads. `SCA_JOBS` sets how many (default: the smaller of
|
||||
4 and the cores; 1 turns it off). Workers are only started when the builds they take off
|
||||
the cell's hands are worth the seconds a worker takes to start (a notebook remembers how
|
||||
long it last took to build), and a notebook that has not changed since it last ran is not
|
||||
started at all. A notebook's state directory is locked between processes while it runs.
|
||||
Desktop-arm benchmark: cold 80.5 s to 34.6 s.
|
||||
- Cache: a cell that does `from lib.dims import A` now depends on the definition of `A` (and
|
||||
what it reads, across local modules), not on the whole file. Editing another name of
|
||||
`lib/dims.py`, a comment, or the formatting leaves the cell cached. A module the analysis
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""A worker process of :mod:`.prefetch`: runs the notebooks it is sent.
|
||||
|
||||
Jobs and answers are length-prefixed pickles on stdin and the original stdout;
|
||||
file descriptor 1 itself is pointed at the null device, so whatever a notebook
|
||||
prints (from Python or from C) cannot get in the way of the protocol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pickle
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
reply = os.fdopen(os.dup(1), "wb")
|
||||
null = os.open(os.devnull, os.O_WRONLY)
|
||||
os.dup2(null, 1)
|
||||
sys.stdout = open(os.devnull, "w") # noqa: SIM115
|
||||
jobs = sys.stdin.buffer
|
||||
from simplecadapi.runtime.runner import run_notebook
|
||||
|
||||
while True:
|
||||
header = jobs.read(8)
|
||||
if len(header) < 8:
|
||||
return 0
|
||||
path, child_id, overrides = pickle.loads(jobs.read(struct.unpack(">Q", header)[0]))
|
||||
try:
|
||||
run_notebook(Path(path), id=child_id, overrides=overrides)
|
||||
answer: tuple[str, str] = ("ok", "")
|
||||
except BaseException as error: # noqa: BLE001 - reported, the parent runs it for itself
|
||||
answer = ("error", f"{type(error).__name__}: {error}"[-2000:])
|
||||
data = pickle.dumps(answer)
|
||||
reply.write(struct.pack(">Q", len(data)) + data)
|
||||
reply.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -41,6 +41,7 @@ from ._marimo import (
|
||||
RunResult,
|
||||
Skip,
|
||||
)
|
||||
from .children import notebook_digest
|
||||
from .config import NotebookConfig
|
||||
from .names import names_tag
|
||||
from .stubs import register_scalar_stubs # noqa: F401 (registers on import)
|
||||
@@ -50,7 +51,6 @@ from .deps import (
|
||||
cell_digest,
|
||||
cell_module_files,
|
||||
read_dependencies,
|
||||
recorded_dependents,
|
||||
state_dir,
|
||||
)
|
||||
|
||||
@@ -133,7 +133,8 @@ class CellDigests:
|
||||
# A cell that runs a child notebook, or reads what one makes, depends
|
||||
# on what the last run recorded; its entries are kept apart so they
|
||||
# can follow the record.
|
||||
self._uses = recorded_dependents(sources)
|
||||
self._uses = frozenset(key for key, deps in self._files.items() if deps.dynamic)
|
||||
self._recorded = read_dependencies(config)
|
||||
self._kernel = kernel
|
||||
self._saved_tag: dict[str, str] = {}
|
||||
if saved:
|
||||
@@ -154,9 +155,32 @@ class CellDigests:
|
||||
if self._kernel and cell_id in self._uses:
|
||||
return None
|
||||
named, fallback = names_tag(self.config.directory, deps.names)
|
||||
tag = ("use" if cell_id in self._uses else "") + self._saved_tag.get(cell_id, "") + named
|
||||
used, unknown = self._used_tag(deps.uses)
|
||||
if unknown:
|
||||
# A notebook this one uses has a part of its own that only a run
|
||||
# can name: depend on what the last run recorded.
|
||||
if self._kernel:
|
||||
return None
|
||||
fallback = fallback | self._recorded
|
||||
tag = ("use" if cell_id in self._uses or unknown else "") + self._saved_tag.get(cell_id, "") + named + used
|
||||
return cell_digest(self.config, deps.files | fallback, tag=tag)
|
||||
|
||||
@staticmethod
|
||||
def _used_tag(paths: frozenset[Path]) -> tuple[str, bool]:
|
||||
"""The digest of the notebooks a cell uses by a literal path, and
|
||||
whether one of them cannot be digested."""
|
||||
|
||||
if not paths:
|
||||
return "", False
|
||||
digests: dict[str, str] = {}
|
||||
for path in sorted(paths):
|
||||
digest = notebook_digest(path)
|
||||
if digest is None:
|
||||
return "", True
|
||||
digests[path.name] = digest
|
||||
encoded = json.dumps(digests, sort_keys=True).encode("utf-8")
|
||||
return "u" + hashlib.sha256(encoded).hexdigest()[:12], False
|
||||
|
||||
|
||||
class CellCache(CachedLifecycle):
|
||||
"""marimo's cell cache, a store per digest, reporting its hits."""
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""What a notebook is, to the notebook that uses it.
|
||||
|
||||
``use("child.py")`` makes a cell depend on the child. When the path is a
|
||||
literal that is known before the cell runs, so the cell is keyed by a digest
|
||||
of the child, and each ``use`` is followed on its own: editing one child
|
||||
recomputes the cells that use it, not those that use its siblings.
|
||||
|
||||
The digest is of what the child *is*: its text, the local modules it imports,
|
||||
the project files it reads, and, in turn, the digest of every notebook it
|
||||
uses. Its text, not its syntax tree: the definition a run makes records where
|
||||
in the file each step is, so a comment or a blank line changes the definition
|
||||
of every notebook that embeds the child, and a cached parent must equal a
|
||||
fresh one.
|
||||
A child that computes a path itself has no such digest (``None``): the caller
|
||||
then depends on whatever a run recorded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .config import read_notebook_config
|
||||
from .deps import file_digest, input_files, local_modules, use_calls
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Parts:
|
||||
syntax: str
|
||||
header: str
|
||||
modules: tuple[Path, ...]
|
||||
inputs: tuple[Path, ...]
|
||||
children: tuple[Path, ...]
|
||||
|
||||
|
||||
_parts_memo: dict[tuple[Path, str | None], _Parts | None] = {}
|
||||
_memo_lock = threading.Lock()
|
||||
|
||||
|
||||
def notebook_digest(path: Path) -> str | None:
|
||||
"""The digest of the notebook at *path* and what it depends on, or ``None``
|
||||
when that cannot be told without running it."""
|
||||
|
||||
return _digest(path.resolve(), frozenset())
|
||||
|
||||
|
||||
def _digest(path: Path, stack: frozenset[Path]) -> str | None:
|
||||
if path in stack:
|
||||
return "cycle" # a run reports it
|
||||
parts = _parts(path)
|
||||
if parts is None:
|
||||
return None
|
||||
here = path.parent
|
||||
children: dict[str, str] = {}
|
||||
for child in parts.children:
|
||||
digest = _digest(child, stack | {path})
|
||||
if digest is None:
|
||||
return None
|
||||
children[_relative(child, here)] = digest
|
||||
payload = {
|
||||
"syntax": parts.syntax,
|
||||
"header": parts.header,
|
||||
"modules": {_relative(item, here): file_digest(item) for item in parts.modules},
|
||||
"inputs": {_relative(item, here): file_digest(item) for item in parts.inputs},
|
||||
"children": children,
|
||||
}
|
||||
encoded = json.dumps(payload, sort_keys=True).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()[:16]
|
||||
|
||||
|
||||
def _parts(path: Path) -> _Parts | None:
|
||||
key = (path, file_digest(path))
|
||||
with _memo_lock:
|
||||
if key in _parts_memo:
|
||||
return _parts_memo[key]
|
||||
parts = _analyse(path)
|
||||
with _memo_lock:
|
||||
if len(_parts_memo) > 512:
|
||||
_parts_memo.clear()
|
||||
_parts_memo[key] = parts
|
||||
return parts
|
||||
|
||||
|
||||
def _analyse(path: Path) -> _Parts | None:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
config = read_notebook_config(path)
|
||||
if config is None:
|
||||
return None
|
||||
calls = use_calls(text)
|
||||
if any(call is None for call in calls):
|
||||
return None
|
||||
return _Parts(
|
||||
syntax=hashlib.sha256(text.encode("utf-8")).hexdigest()[:16],
|
||||
header=repr((config.id, config.revision, config.tolerance_profile, config.inputs)),
|
||||
modules=tuple(sorted(local_modules(config, [text]).values())),
|
||||
inputs=tuple(sorted(input_files(config))),
|
||||
children=tuple(sorted({(path.parent / call).resolve() for call in calls if call is not None})),
|
||||
)
|
||||
|
||||
|
||||
def _relative(path: Path, start: Path) -> str:
|
||||
try:
|
||||
return Path(os.path.relpath(path, start)).as_posix()
|
||||
except ValueError: # another drive on Windows
|
||||
return path.as_posix()
|
||||
|
||||
|
||||
__all__ = ["notebook_digest"]
|
||||
@@ -92,6 +92,49 @@ def state_dir(config: NotebookConfig) -> Path:
|
||||
return notebook_output_dir(config.path) / "simplecad" / config.path.name / config.id
|
||||
|
||||
|
||||
_held: set[str] = set()
|
||||
_held_lock = threading.Lock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def exclusive(config: NotebookConfig) -> Iterator[None]:
|
||||
"""Hold the notebook's state directory against other processes.
|
||||
|
||||
Worker processes may run the same notebook at once (two notebooks that
|
||||
use a third), and a run prunes the cache directories it did not use. The
|
||||
lock is between processes only: a run of a notebook this process is
|
||||
already running (a ``use`` cycle, an editor next to a headless run) goes
|
||||
on without it, as it always did. Without ``fcntl`` (Windows) it is a
|
||||
no-op.
|
||||
"""
|
||||
|
||||
directory = state_dir(config)
|
||||
key = str(directory)
|
||||
with _held_lock:
|
||||
nested = key in _held
|
||||
_held.add(key)
|
||||
handle = None
|
||||
try:
|
||||
if not nested:
|
||||
try:
|
||||
import fcntl
|
||||
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
handle = open(directory / "run.lock", "a") # noqa: SIM115
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
except (ImportError, OSError):
|
||||
if handle is not None:
|
||||
handle.close()
|
||||
handle = None
|
||||
yield
|
||||
finally:
|
||||
if handle is not None:
|
||||
handle.close() # releases the lock
|
||||
if not nested:
|
||||
with _held_lock:
|
||||
_held.discard(key)
|
||||
|
||||
|
||||
def deps_file(config: NotebookConfig) -> Path:
|
||||
return state_dir(config) / "deps.json"
|
||||
|
||||
@@ -312,9 +355,18 @@ class CellDeps:
|
||||
|
||||
files: frozenset[Path]
|
||||
names: frozenset[NameDep] = frozenset()
|
||||
uses: frozenset[Path] = frozenset()
|
||||
"""Notebooks the cell runs with ``use``, by a literal path (see :mod:`.children`)."""
|
||||
dynamic: bool = False
|
||||
"""The cell runs a notebook whose path is computed: only a run knows which."""
|
||||
|
||||
def __or__(self, other: CellDeps) -> CellDeps:
|
||||
return CellDeps(self.files | other.files, self.names | other.names)
|
||||
return CellDeps(
|
||||
self.files | other.files,
|
||||
self.names | other.names,
|
||||
self.uses | other.uses,
|
||||
self.dynamic or other.dynamic,
|
||||
)
|
||||
|
||||
|
||||
def cell_module_files(
|
||||
@@ -411,9 +463,14 @@ def cell_module_files(
|
||||
named.add(NameDep(module_name, original, frozenset(coarse)))
|
||||
else:
|
||||
files |= coarse
|
||||
if calls_use(cell.code):
|
||||
calls = use_calls(cell.code)
|
||||
dynamic = any(call is None for call in calls)
|
||||
if dynamic:
|
||||
files |= recorded_files
|
||||
result[cell.key] = CellDeps(frozenset(files), frozenset(named))
|
||||
uses = frozenset(
|
||||
(root / call).resolve() for call in calls if call is not None
|
||||
)
|
||||
result[cell.key] = CellDeps(frozenset(files), frozenset(named), uses, dynamic)
|
||||
|
||||
# marimo keys a cell that reads an object (a solid, a part) by the code of
|
||||
# the cells that made it, not by its value: a file an ancestor depends on
|
||||
@@ -565,6 +622,27 @@ def _loaded_names_of(node: ast.AST) -> set[str]:
|
||||
}
|
||||
|
||||
|
||||
def use_calls(source: str) -> list[str | None]:
|
||||
"""The paths of the ``use(...)`` calls in *source*: the literal, or ``None``
|
||||
where it is computed (or the source cannot be read)."""
|
||||
|
||||
try:
|
||||
tree = ast.parse(source)
|
||||
except (SyntaxError, ValueError):
|
||||
return [None]
|
||||
paths: list[str | None] = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Call):
|
||||
function = node.func
|
||||
if (isinstance(function, ast.Name) and function.id == "use") or (
|
||||
isinstance(function, ast.Attribute) and function.attr == "use"
|
||||
):
|
||||
first = node.args[0] if node.args else None
|
||||
literal = isinstance(first, ast.Constant) and isinstance(first.value, str)
|
||||
paths.append(first.value if literal else None) # type: ignore[union-attr]
|
||||
return paths
|
||||
|
||||
|
||||
def calls_use(source: str) -> bool:
|
||||
try:
|
||||
tree = ast.parse(source)
|
||||
@@ -668,12 +746,14 @@ __all__ = [
|
||||
"NameDep",
|
||||
"ancestors",
|
||||
"calls_use",
|
||||
"use_calls",
|
||||
"cell_digest",
|
||||
"cell_module_files",
|
||||
"dependency_digest",
|
||||
"file_digest",
|
||||
"recorded_dependents",
|
||||
"deps_file",
|
||||
"exclusive",
|
||||
"input_files",
|
||||
"local_modules",
|
||||
"read_dependencies",
|
||||
|
||||
@@ -45,6 +45,7 @@ from ._marimo import (
|
||||
RunResult,
|
||||
safe_get_context,
|
||||
)
|
||||
from . import prefetch
|
||||
from .cache import CellCache, CellDigests, cache_root, prune
|
||||
from .cells import NotebookCells, bound_requirements
|
||||
from .config import NotebookConfig, read_notebook_config
|
||||
@@ -116,6 +117,7 @@ class SimpleCadExecutor:
|
||||
if restored is not _MISS:
|
||||
return restored
|
||||
cache = cache if restored is _MISS else None
|
||||
_prefetch(scope, cell, glbls)
|
||||
try:
|
||||
with _cell_recording(scope, cell, glbls):
|
||||
result = self._delegate.execute_cell(cell, glbls)
|
||||
@@ -134,6 +136,7 @@ class SimpleCadExecutor:
|
||||
restored = _restore(cache, cell, glbls)
|
||||
if restored is not _MISS:
|
||||
return restored
|
||||
_prefetch(scope, cell, glbls)
|
||||
try:
|
||||
with _cell_recording(scope, cell, glbls):
|
||||
result = await self._delegate.execute_cell_async(cell, glbls)
|
||||
@@ -160,6 +163,15 @@ class SimpleCadExecutor:
|
||||
return _kernel_cache(scope, cell)
|
||||
|
||||
|
||||
def _prefetch(scope: NotebookScope, cell: CellImpl, glbls: MutableGlobals) -> None:
|
||||
"""Start the notebooks the cell is about to ``use`` in worker processes."""
|
||||
|
||||
try:
|
||||
prefetch.start(scope.config.directory, cell.code, glbls, scope.config.id)
|
||||
except Exception: # noqa: BLE001 - an optimisation, never a reason for a cell to fail
|
||||
pass
|
||||
|
||||
|
||||
_MISS = object()
|
||||
"""Returned by :func:`_restore` when the cell must run."""
|
||||
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
"""Run the notebooks a cell uses side by side.
|
||||
|
||||
OpenCASCADE does not let go of the interpreter lock (two threads building
|
||||
the same kind of model are no faster than one), so the parallelism is in
|
||||
processes. When a cell is about to run, the ``use`` calls it is going to make
|
||||
that can be told beforehand (:func:`plan_uses`: literal paths, arguments made
|
||||
of values the cell reads) are handed to worker processes. A worker runs the
|
||||
notebook headlessly, which fills its cell cache; the cell then makes its
|
||||
``use`` calls as it always did, waits for the worker of each (:func:`wait`),
|
||||
and finds the notebook built. Nothing about the result depends on the
|
||||
workers: a failed or missing one only means the call runs the notebook itself.
|
||||
|
||||
``SCA_JOBS`` sets how many workers (default: the smaller of 4 and the number
|
||||
of cores; 1 turns this off). A notebook that has not changed since it last
|
||||
ran, with the same overrides, is not started: it is in the cache already.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..artifacts.canonical import validate_logical_id
|
||||
from .children import notebook_digest
|
||||
from .config import read_notebook_config
|
||||
from .deps import state_dir, use_calls
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# What a cell is going to use
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UseRequest:
|
||||
path: Path
|
||||
id: str | None
|
||||
overrides: Mapping[str, Any]
|
||||
|
||||
@property
|
||||
def key(self) -> tuple[str, str | None, str]:
|
||||
return request_key(self.path, self.id, self.overrides)
|
||||
|
||||
|
||||
def request_key(path: Path, id: str | None, overrides: Mapping[str, Any]) -> tuple[str, str | None, str]:
|
||||
return (str(Path(path).resolve()), id, repr(sorted(overrides.items(), key=lambda item: item[0])))
|
||||
|
||||
|
||||
class _Unknown(Exception):
|
||||
"""An expression the plan cannot evaluate before the cell runs."""
|
||||
|
||||
|
||||
_CALLS = {"float": float, "int": int, "str": str, "bool": bool, "round": round, "abs": abs, "min": min, "max": max}
|
||||
_BINARY = {
|
||||
ast.Add: lambda a, b: a + b,
|
||||
ast.Sub: lambda a, b: a - b,
|
||||
ast.Mult: lambda a, b: a * b,
|
||||
ast.Div: lambda a, b: a / b,
|
||||
ast.FloorDiv: lambda a, b: a // b,
|
||||
ast.Mod: lambda a, b: a % b,
|
||||
ast.Pow: lambda a, b: a**b,
|
||||
}
|
||||
_COMPARE = {
|
||||
ast.Eq: lambda a, b: a == b,
|
||||
ast.NotEq: lambda a, b: a != b,
|
||||
ast.Lt: lambda a, b: a < b,
|
||||
ast.LtE: lambda a, b: a <= b,
|
||||
ast.Gt: lambda a, b: a > b,
|
||||
ast.GtE: lambda a, b: a >= b,
|
||||
}
|
||||
_NOT_RUN_HERE = (
|
||||
ast.IfExp, ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.FunctionDef,
|
||||
ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda, ast.ListComp, ast.SetComp,
|
||||
ast.DictComp, ast.GeneratorExp, ast.Match,
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
def _evaluate(node: ast.expr, scope: Mapping[str, Any], notebook_id: str | None = None) -> Any:
|
||||
"""The value of *node*, for the few expression forms a plan can trust."""
|
||||
|
||||
if isinstance(node, ast.Constant):
|
||||
return node.value
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and not node.args
|
||||
and not node.keywords
|
||||
and (
|
||||
(isinstance(node.func, ast.Name) and node.func.id == "notebook_id")
|
||||
or (isinstance(node.func, ast.Attribute) and node.func.attr == "notebook_id")
|
||||
)
|
||||
):
|
||||
if notebook_id is None:
|
||||
raise _Unknown
|
||||
return notebook_id
|
||||
if isinstance(node, ast.Name):
|
||||
if node.id in scope:
|
||||
return scope[node.id]
|
||||
raise _Unknown
|
||||
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd, ast.Not)):
|
||||
value = _evaluate(node.operand, scope, notebook_id)
|
||||
return {ast.USub: lambda: -value, ast.UAdd: lambda: +value, ast.Not: lambda: not value}[type(node.op)]()
|
||||
if isinstance(node, ast.BinOp) and type(node.op) in _BINARY:
|
||||
return _BINARY[type(node.op)](_evaluate(node.left, scope, notebook_id), _evaluate(node.right, scope, notebook_id))
|
||||
if isinstance(node, ast.Compare) and len(node.ops) == 1 and type(node.ops[0]) in _COMPARE:
|
||||
return _COMPARE[type(node.ops[0])](_evaluate(node.left, scope, notebook_id), _evaluate(node.comparators[0], scope, notebook_id))
|
||||
if isinstance(node, ast.IfExp):
|
||||
return _evaluate(node.body if _evaluate(node.test, scope, notebook_id) else node.orelse, scope, notebook_id)
|
||||
if isinstance(node, (ast.Tuple, ast.List)):
|
||||
items = [_evaluate(item, scope, notebook_id) for item in node.elts]
|
||||
return tuple(items) if isinstance(node, ast.Tuple) else items
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id in _CALLS
|
||||
and node.func.id not in scope
|
||||
and not node.keywords
|
||||
):
|
||||
return _CALLS[node.func.id](*[_evaluate(arg, scope, notebook_id) for arg in node.args])
|
||||
raise _Unknown
|
||||
|
||||
|
||||
def plan_uses(
|
||||
code: str, scope: Mapping[str, Any], directory: Path, notebook_id: str | None = None
|
||||
) -> list[UseRequest]:
|
||||
"""The ``use`` calls *code* will make that are known before it runs.
|
||||
|
||||
A call counts when its path is a literal and every argument can be worked
|
||||
out from *scope* (the values the cell reads) with arithmetic and
|
||||
``float(...)``-like calls; and when the cell is certain to make it (not
|
||||
under a condition, in a loop or in a function).
|
||||
"""
|
||||
|
||||
try:
|
||||
tree = ast.parse(code)
|
||||
except (SyntaxError, ValueError):
|
||||
return []
|
||||
requests: dict[tuple[str, str | None, str], UseRequest] = {}
|
||||
|
||||
def visit(node: ast.AST) -> None:
|
||||
if isinstance(node, _NOT_RUN_HERE):
|
||||
return
|
||||
if isinstance(node, ast.Call) and _is_use(node):
|
||||
request = _request(node, scope, directory, notebook_id)
|
||||
if request is not None:
|
||||
requests.setdefault(request.key, request)
|
||||
for child in ast.iter_child_nodes(node):
|
||||
visit(child)
|
||||
|
||||
visit(tree)
|
||||
return list(requests.values())
|
||||
|
||||
|
||||
def _is_use(node: ast.Call) -> bool:
|
||||
function = node.func
|
||||
return (isinstance(function, ast.Name) and function.id == "use") or (
|
||||
isinstance(function, ast.Attribute) and function.attr == "use"
|
||||
)
|
||||
|
||||
|
||||
def _request(
|
||||
node: ast.Call, scope: Mapping[str, Any], directory: Path, notebook_id: str | None
|
||||
) -> UseRequest | None:
|
||||
if not node.args or not (isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str)):
|
||||
return None
|
||||
if len(node.args) != 1 or any(keyword.arg is None for keyword in node.keywords):
|
||||
return None
|
||||
try:
|
||||
values = {keyword.arg: _evaluate(keyword.value, scope, notebook_id) for keyword in node.keywords}
|
||||
except Exception: # noqa: BLE001 - anything wrong here only means "not planned"
|
||||
return None
|
||||
child_id = values.pop("id", None)
|
||||
if child_id is not None and not isinstance(child_id, str):
|
||||
return None
|
||||
path = Path(node.args[0].value)
|
||||
return UseRequest((path if path.is_absolute() else directory / path).resolve(), child_id, values)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Has the notebook changed since it last ran?
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _signature(path: Path, config_id: str, overrides: Mapping[str, Any]) -> str | None:
|
||||
digest = notebook_digest(path)
|
||||
if digest is None:
|
||||
return None
|
||||
payload = repr((digest, config_id, sorted(overrides.items(), key=lambda item: item[0])))
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:20]
|
||||
|
||||
|
||||
def _stamp_file(config: Any) -> Path:
|
||||
return state_dir(config) / "stamp.json"
|
||||
|
||||
|
||||
def _read_stamp(config: Any) -> dict[str, Any]:
|
||||
try:
|
||||
data = json.loads(_stamp_file(config).read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def write_stamp(config: Any, overrides: Mapping[str, Any], seconds: float | None = None) -> None:
|
||||
"""Record that *config*'s notebook has just run to the end with *overrides*.
|
||||
|
||||
*seconds* is how long the run took when it built something; a run that
|
||||
only restored keeps the figure of the last build (it says what the
|
||||
notebook costs, which is what decides whether a worker is worth it).
|
||||
"""
|
||||
|
||||
signature = _signature(config.path, config.id, overrides)
|
||||
target = _stamp_file(config)
|
||||
try:
|
||||
if signature is None:
|
||||
target.unlink(missing_ok=True)
|
||||
return
|
||||
previous = _read_stamp(config)
|
||||
if seconds is None:
|
||||
seconds = previous.get("seconds")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json.dumps({"signature": signature, "seconds": seconds}), encoding="utf-8")
|
||||
except OSError:
|
||||
pass # the stamp only saves work
|
||||
|
||||
|
||||
def _expected_seconds(request: UseRequest) -> float | None:
|
||||
"""How long the notebook has to be built, if it has to be: ``None`` when it
|
||||
does not (it is in the cache), ``inf`` when nobody knows how long."""
|
||||
|
||||
config = read_notebook_config(request.path)
|
||||
if config is None:
|
||||
return None # the call itself will say what is wrong with it
|
||||
try:
|
||||
if request.id is not None:
|
||||
config = replace(config, id=validate_logical_id(request.id, "/run/id"))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
signature = _signature(config.path, config.id, request.overrides)
|
||||
stamp = _read_stamp(config)
|
||||
if signature is not None and stamp.get("signature") == signature:
|
||||
return None
|
||||
seconds = stamp.get("seconds")
|
||||
return float(seconds) if isinstance(seconds, (int, float)) else float("inf")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COLD_WORTH = 4.0
|
||||
"""Seconds a start of the workers has to save to be done (a worker imports the
|
||||
kernel libraries: a few seconds)."""
|
||||
_WARM_WORTH = 0.5
|
||||
"""...and once they are running."""
|
||||
|
||||
_stats: dict[str, list[str]] = {"submitted": []}
|
||||
_pending: dict[tuple[str, str | None, str], Future[None]] = {}
|
||||
_lock = threading.Lock()
|
||||
_pool: _Pool | None = None
|
||||
|
||||
|
||||
def jobs() -> int:
|
||||
raw = os.environ.get("SCA_JOBS")
|
||||
if raw is not None:
|
||||
try:
|
||||
return max(1, int(raw))
|
||||
except ValueError:
|
||||
return 1
|
||||
return max(1, min(4, os.cpu_count() or 1))
|
||||
|
||||
|
||||
def stats() -> dict[str, list[str]]:
|
||||
return {"submitted": list(_stats["submitted"])}
|
||||
|
||||
|
||||
def reset_stats() -> None:
|
||||
_stats["submitted"].clear()
|
||||
|
||||
|
||||
def start(directory: Path, code: str, scope: Mapping[str, Any], notebook_id: str | None = None) -> None:
|
||||
"""Hand the ``use`` calls of a cell about to run to the workers."""
|
||||
|
||||
global _pool
|
||||
size = jobs()
|
||||
if size < 2 or "use" not in code:
|
||||
return
|
||||
if not use_calls(code):
|
||||
return
|
||||
planned = [(request, _expected_seconds(request)) for request in plan_uses(code, scope, directory, notebook_id)]
|
||||
wanted = [(request, seconds) for request, seconds in planned if seconds is not None]
|
||||
if len(wanted) < 2:
|
||||
return # nothing to do beside the cell's own first call
|
||||
# The longest build goes on in this process; what the workers take off its
|
||||
# hands has to be worth more than starting them.
|
||||
seconds = sorted((value for _, value in wanted), reverse=True)
|
||||
saved = sum(seconds[1:])
|
||||
if saved < (_WARM_WORTH if _pool is not None and _pool.size == size else _COLD_WORTH):
|
||||
return
|
||||
requests = [request for request, _ in wanted]
|
||||
with _lock:
|
||||
if _pool is None or _pool.size != size:
|
||||
_pool = _Pool(size)
|
||||
for request in requests:
|
||||
existing = _pending.get(request.key)
|
||||
if existing is not None and not existing.done():
|
||||
continue
|
||||
_pending[request.key] = _pool.submit(request)
|
||||
_stats["submitted"].append(str(request.path))
|
||||
|
||||
|
||||
def wait(path: Path, id: str | None, overrides: Mapping[str, Any]) -> None:
|
||||
"""Wait for the worker that is building this notebook, if one is."""
|
||||
|
||||
with _lock:
|
||||
future = _pending.pop(request_key(path, id, overrides), None)
|
||||
if future is not None:
|
||||
try:
|
||||
future.result()
|
||||
except Exception: # noqa: BLE001 - the call runs the notebook and reports for itself
|
||||
pass
|
||||
|
||||
|
||||
def shutdown() -> None:
|
||||
global _pool
|
||||
with _lock:
|
||||
pool, _pool = _pool, None
|
||||
_pending.clear()
|
||||
if pool is not None:
|
||||
pool.close()
|
||||
|
||||
|
||||
class _Pool:
|
||||
"""Threads that each own a worker process, started when first needed."""
|
||||
|
||||
def __init__(self, size: int) -> None:
|
||||
self.size = size
|
||||
self._executor = ThreadPoolExecutor(size, thread_name_prefix="sca-prefetch")
|
||||
self._local = threading.local()
|
||||
self._workers: list[subprocess.Popen[bytes]] = []
|
||||
self._workers_lock = threading.Lock()
|
||||
|
||||
def submit(self, request: UseRequest) -> Future[None]:
|
||||
return self._executor.submit(self._run, request)
|
||||
|
||||
def _run(self, request: UseRequest) -> None:
|
||||
worker = getattr(self._local, "worker", None)
|
||||
if worker is None or worker.poll() is not None:
|
||||
worker = self._local.worker = self._spawn()
|
||||
try:
|
||||
_send(worker.stdin, (str(request.path), request.id, dict(request.overrides))) # type: ignore[arg-type]
|
||||
reply = _receive(worker.stdout) # type: ignore[arg-type]
|
||||
except Exception:
|
||||
self._local.worker = None
|
||||
worker.kill()
|
||||
raise
|
||||
if reply[0] != "ok":
|
||||
raise RuntimeError(reply[1])
|
||||
|
||||
def _spawn(self) -> subprocess.Popen[bytes]:
|
||||
worker = subprocess.Popen(
|
||||
[sys.executable, "-m", "simplecadapi.runtime._prefetch_worker"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
env={**os.environ, "SCA_JOBS": "1"}, # a worker does not start workers
|
||||
)
|
||||
with self._workers_lock:
|
||||
self._workers.append(worker)
|
||||
return worker
|
||||
|
||||
def close(self) -> None:
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
with self._workers_lock:
|
||||
workers, self._workers = self._workers, []
|
||||
for worker in workers:
|
||||
try:
|
||||
worker.kill()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _send(stream: Any, value: Any) -> None:
|
||||
data = pickle.dumps(value)
|
||||
stream.write(struct.pack(">Q", len(data)) + data)
|
||||
stream.flush()
|
||||
|
||||
|
||||
def _receive(stream: Any) -> Any:
|
||||
header = stream.read(8)
|
||||
if len(header) < 8:
|
||||
raise EOFError("the worker ended")
|
||||
(size,) = struct.unpack(">Q", header)
|
||||
return pickle.loads(stream.read(size))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"UseRequest",
|
||||
"jobs",
|
||||
"plan_uses",
|
||||
"request_key",
|
||||
"reset_stats",
|
||||
"shutdown",
|
||||
"start",
|
||||
"stats",
|
||||
"wait",
|
||||
"write_stamp",
|
||||
]
|
||||
@@ -24,6 +24,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
@@ -53,11 +54,13 @@ from ._marimo import (
|
||||
safe_get_context,
|
||||
suppress_warnings_logs,
|
||||
)
|
||||
from . import prefetch
|
||||
from .cache import CellCache, CellDigests, cache_root, prune
|
||||
from .cells import NotebookCells
|
||||
from .config import NotebookConfig, NotebookConfigError, read_notebook_config
|
||||
from .deps import (
|
||||
local_modules,
|
||||
exclusive,
|
||||
read_dependencies,
|
||||
record_dependencies,
|
||||
recording_dependencies,
|
||||
@@ -163,7 +166,7 @@ def run_notebook(
|
||||
# Errors still come through. "marimo" is the logger every marimo
|
||||
# module shares (lazy.py's record shows its file name, "lazy", which
|
||||
# is not the logger's name).
|
||||
with suppress_warnings_logs("marimo"):
|
||||
with suppress_warnings_logs("marimo"), exclusive(config):
|
||||
return _run(config, values, cache=cache, through=through, product=product)
|
||||
|
||||
# marimo keeps its runtime context per thread, and a notebook run needs
|
||||
@@ -188,6 +191,7 @@ def use(path: str | Path, /, *, id: str | None = None, **overrides: Any) -> Prod
|
||||
target = Path(path)
|
||||
if not target.is_absolute():
|
||||
target = _calling_directory() / target
|
||||
prefetch.wait(target, id, overrides) # a worker may be building it
|
||||
child = run_notebook(target, id=id, overrides=overrides)
|
||||
# use() never asks for a partial run, so the product is always there.
|
||||
if child.product is None:
|
||||
@@ -209,6 +213,8 @@ def _run(
|
||||
through: int | None = None,
|
||||
product: bool = True,
|
||||
) -> NotebookRun:
|
||||
started = time.perf_counter()
|
||||
given = dict(overrides) # (``overrides`` becomes the run's values below)
|
||||
app = load_notebook(config.path, cell_prefix=_cell_prefix(config))
|
||||
if app is None:
|
||||
raise NotebookConfigError(f"{config.path}: the notebook has no cells")
|
||||
@@ -310,6 +316,9 @@ def _run(
|
||||
)
|
||||
for cell in ordered
|
||||
)
|
||||
if product and through is None and cache:
|
||||
built = any(report.status == "ran" and report.name != "setup" for report in reports)
|
||||
prefetch.write_stamp(config, given, time.perf_counter() - started if built else None)
|
||||
return NotebookRun(
|
||||
config=config,
|
||||
product=product_value,
|
||||
|
||||
@@ -11,3 +11,8 @@ import os
|
||||
# with the agent-facing default.
|
||||
os.environ.setdefault("SCA_NO_DIAGNOSTIC_RENDER", "1")
|
||||
os.environ.setdefault("SCA_DIAGNOSTIC_RENDER_BUDGET", "60")
|
||||
|
||||
# Worker processes for the notebooks a cell uses are for the tests that are
|
||||
# about them (they set SCA_JOBS themselves): starting several interpreters per
|
||||
# run would only slow the rest of the suite down.
|
||||
os.environ.setdefault("SCA_JOBS", "1")
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""One process at a time in a notebook's state directory.
|
||||
|
||||
Workers (see ``prefetch``) may run the same notebook at once, and a run
|
||||
prunes the cache directories it did not use. The lock is between processes:
|
||||
inside one, a nested run of the same notebook (a ``use`` cycle) goes on as it
|
||||
always did, and is reported, not waited for.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
|
||||
from simplecadapi.runtime.config import read_notebook_config
|
||||
from simplecadapi.runtime.deps import exclusive
|
||||
from test_notebook_runtime import NotebookTestCase, cell, notebook_text
|
||||
|
||||
|
||||
class TestStateLock(NotebookTestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.path = self.write("n.py", notebook_text(cell("x = 1", returns="x")))
|
||||
self.config = read_notebook_config(self.path)
|
||||
|
||||
def test_a_nested_run_of_the_same_notebook_is_not_waited_for(self) -> None:
|
||||
with exclusive(self.config):
|
||||
with exclusive(self.config):
|
||||
pass
|
||||
|
||||
def test_another_process_waits_for_the_holder(self) -> None:
|
||||
holder = textwrap.dedent(
|
||||
f"""
|
||||
import sys, time
|
||||
from simplecadapi.runtime.config import read_notebook_config
|
||||
from simplecadapi.runtime.deps import exclusive
|
||||
with exclusive(read_notebook_config({str(self.path)!r})):
|
||||
print("locked", flush=True)
|
||||
time.sleep(1.5)
|
||||
"""
|
||||
)
|
||||
process = subprocess.Popen([sys.executable, "-c", holder], stdout=subprocess.PIPE, text=True)
|
||||
try:
|
||||
self.assertEqual(process.stdout.readline().strip(), "locked")
|
||||
started = time.monotonic()
|
||||
with exclusive(self.config):
|
||||
waited = time.monotonic() - started
|
||||
self.assertGreater(waited, 0.8)
|
||||
finally:
|
||||
process.wait(timeout=30)
|
||||
@@ -0,0 +1,110 @@
|
||||
"""`use` in the cache: a cell depends on the notebooks it uses, one by one.
|
||||
|
||||
The path of a ``use`` call is usually a literal, so which notebooks a cell
|
||||
depends on is known before it runs: the cell is keyed by what each of them is
|
||||
(its code, not its comments or layout; the modules it imports; the notebooks
|
||||
it uses in turn). What is computed at run time is recorded by a run, as
|
||||
before.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from simplecadapi.runtime import run_notebook
|
||||
from simplecadapi.runtime._marimo import InternalApp, load_notebook
|
||||
from simplecadapi.runtime.cache import CellDigests
|
||||
from simplecadapi.runtime.config import read_notebook_config
|
||||
from simplecadapi.runtime.runner import _cell_prefix
|
||||
from test_notebook_runtime import NotebookTestCase, cell, notebook_text
|
||||
|
||||
|
||||
def child(name: str, width: float, note: str = "") -> str:
|
||||
return notebook_text(
|
||||
cell(f"width = {width}{note}", returns="width"),
|
||||
cell("block = scad.make_box_rsolid(width, 4.0, 4.0)", reads="width", returns="block"),
|
||||
cell(f"part = scad.Part(part_id='{name}', body=block)", reads="block", returns="part"),
|
||||
table=f'id = "{name}"',
|
||||
)
|
||||
|
||||
|
||||
ASSEMBLE = """
|
||||
_asm = scad.make_assembly_rassembly(assembly_id="top", name="top")
|
||||
for _key, _item in (("a", a), ("b", b)):
|
||||
_asm = scad.add_component_rassembly(
|
||||
assembly=_asm, item=_item, component_id=_key, placement=scad.identity_placement_rplacement()
|
||||
)
|
||||
top = _asm
|
||||
"""
|
||||
|
||||
|
||||
def parent(first: str = 'scad.use("child_a.py")', second: str = 'scad.use("child_b.py")', *, before: str = "") -> str:
|
||||
cells = [cell(f"{before}\na = {first}" if before else f"a = {first}", returns="a")]
|
||||
cells += [cell(f"b = {second}", returns="b"), cell(ASSEMBLE, reads="a, b", returns="top")]
|
||||
return notebook_text(*cells, table='id = "top"')
|
||||
|
||||
|
||||
class TestStaticUse(NotebookTestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.write("child_a.py", child("child_a", 10.0))
|
||||
self.write("child_b.py", child("child_b", 6.0))
|
||||
self.top = self.write("top.py", parent())
|
||||
|
||||
def edited(self) -> list[str]:
|
||||
run = run_notebook(self.top)
|
||||
fresh = run_notebook(self.top, cache=False)
|
||||
self.assertEqual(run.definition.content_hash, fresh.definition.content_hash)
|
||||
return self.statuses(run)
|
||||
|
||||
def test_editing_one_child_redoes_only_the_cell_that_uses_it(self) -> None:
|
||||
run_notebook(self.top)
|
||||
self.write("child_a.py", child("child_a", 12.0))
|
||||
# setup, use a, use b, the assembly
|
||||
self.assertEqual(self.edited(), ["ran", "ran", "cached", "ran"])
|
||||
|
||||
def test_a_comment_in_a_child_is_a_change_like_any_other(self) -> None:
|
||||
# The definition records where in the file each step is: a parent that
|
||||
# embeds the child must equal a fresh run, comment or not.
|
||||
run_notebook(self.top)
|
||||
self.write("child_a.py", child("child_a", 10.0, note=" # the width, in mm"))
|
||||
self.assertEqual(self.edited(), ["ran", "ran", "cached", "ran"])
|
||||
|
||||
def test_a_module_a_child_imports_is_followed(self) -> None:
|
||||
self.write("lib/size.py", "SIZE = 5.0\n")
|
||||
text = child("child_a", 10.0).replace(
|
||||
"with app.setup:\n import simplecadapi as scad\n",
|
||||
"with app.setup:\n import simplecadapi as scad\n from lib.size import SIZE\n",
|
||||
).replace("make_box_rsolid(width, 4.0, 4.0)", "make_box_rsolid(width, SIZE, 4.0)")
|
||||
self.write("child_a.py", text)
|
||||
run_notebook(self.top)
|
||||
self.write("lib/size.py", "SIZE = 6.0\n")
|
||||
self.assertEqual(self.edited(), ["ran", "ran", "cached", "ran"])
|
||||
|
||||
def test_a_grandchild_is_followed_through_the_child(self) -> None:
|
||||
self.write("deep.py", child("deep", 3.0))
|
||||
text = child("child_a", 10.0).replace(
|
||||
"block = scad.make_box_rsolid(width, 4.0, 4.0)",
|
||||
'deep = scad.use("deep.py")\n block = scad.make_box_rsolid(width, 4.0, 4.0)',
|
||||
)
|
||||
self.write("child_a.py", text)
|
||||
run_notebook(self.top)
|
||||
self.write("deep.py", child("deep", 4.0))
|
||||
self.assertEqual(self.edited(), ["ran", "ran", "cached", "ran"])
|
||||
|
||||
def test_a_path_computed_at_run_time_is_still_followed(self) -> None:
|
||||
self.write("top.py", parent(first='scad.use(_name + ".py")', before='_name = "child_a"'))
|
||||
run_notebook(self.top)
|
||||
self.write("child_a.py", child("child_a", 12.0))
|
||||
statuses = self.edited()
|
||||
self.assertEqual(statuses[1], "ran")
|
||||
self.assertEqual(statuses[3], "ran")
|
||||
|
||||
def test_the_kernel_keys_a_static_use_cell_and_runs_a_dynamic_one_live(self) -> None:
|
||||
for source, expected in ((parent(), True), (parent(first='scad.use(_name + ".py")', before='_name = "child_a"'), False)):
|
||||
path = self.write("probe.py", source)
|
||||
config = read_notebook_config(path)
|
||||
internal = InternalApp(load_notebook(path, cell_prefix=_cell_prefix(config)))
|
||||
ordered = [internal.graph.cells[i] for i in internal.cell_manager.cell_ids() if i in internal.graph.cells]
|
||||
setup = internal.graph.cells.get(internal.cell_manager.setup_cell_id)
|
||||
digests = CellDigests(config, ordered, setup, kernel=True)
|
||||
first_use = next(c for c in ordered if "use(" in c.code)
|
||||
self.assertEqual(digests.digest(first_use.cell_id) is not None, expected)
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Running the notebooks a cell uses side by side.
|
||||
|
||||
OpenCASCADE does not let go of the interpreter lock, so threads would not
|
||||
help: the ``use`` calls of a cell that is about to run are started in worker
|
||||
processes, which only fill the notebooks' cell caches. The cell then runs its
|
||||
``use`` calls as it always did, and finds the work done. Whatever the workers
|
||||
do (or fail to do), the result is the one a serial run gives.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from simplecadapi.runtime import prefetch, run_notebook
|
||||
from test_notebook_runtime import NotebookTestCase, notebook_text
|
||||
from test_use_digests import ASSEMBLE, child, parent
|
||||
|
||||
|
||||
class TestPlan(unittest.TestCase):
|
||||
root = Path("/proj")
|
||||
|
||||
def plan(self, code: str, scope: dict | None = None):
|
||||
return prefetch.plan_uses(code, scope or {}, self.root)
|
||||
|
||||
def test_literal_paths_and_values_known_before_the_cell_runs(self) -> None:
|
||||
requests = self.plan(
|
||||
'a = scad.use("child_a.py")\nb = scad.use("sub/b.py", id="x", WIDTH=float(W), N=3)',
|
||||
{"W": 4},
|
||||
)
|
||||
self.assertEqual([r.path for r in requests], [self.root / "child_a.py", self.root / "sub/b.py"])
|
||||
self.assertEqual((requests[1].id, requests[1].overrides), ("x", {"WIDTH": 4.0, "N": 3}))
|
||||
|
||||
def test_the_id_of_the_running_notebook_is_known(self) -> None:
|
||||
requests = prefetch.plan_uses(
|
||||
'a = scad.use("c.py", id=scad.notebook_id() + "-body", LENGTH=float(L))', {"L": 2}, self.root, notebook_id="arm"
|
||||
)
|
||||
self.assertEqual([(r.id, dict(r.overrides)) for r in requests], [("arm-body", {"LENGTH": 2.0})])
|
||||
self.assertEqual(self.plan('a = scad.use("c.py", id=scad.notebook_id())'), []) # not told which
|
||||
|
||||
def test_what_cannot_be_known_is_left_to_the_cell(self) -> None:
|
||||
for code in (
|
||||
'a = scad.use(name + ".py")', # computed path
|
||||
'a = scad.use("c.py") if flag else None', # may not run
|
||||
'for n in names:\n scad.use("c.py")', # a loop
|
||||
'a = scad.use("c.py", LENGTH=other.size())', # a call the plan will not make
|
||||
'a = scad.use("c.py", LENGTH=missing)', # a name the cell defines itself
|
||||
"def f():\n return scad.use('c.py')", # not run by the cell
|
||||
):
|
||||
with self.subTest(code=code):
|
||||
self.assertEqual(self.plan(code, {"flag": True, "names": [1], "other": object()}), [])
|
||||
|
||||
def test_two_requests_for_one_run_are_one(self) -> None:
|
||||
requests = self.plan('a = scad.use("c.py", W=1.0)\nb = scad.use("c.py", W=1.0)\nc = scad.use("c.py", W=2.0)')
|
||||
self.assertEqual(len({r.key for r in requests}), 2)
|
||||
|
||||
|
||||
class TestPrefetchedRun(NotebookTestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
patcher = mock.patch.dict(os.environ, {"SCA_JOBS": "2"})
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
self.addCleanup(prefetch.shutdown)
|
||||
prefetch.reset_stats()
|
||||
self.write("child_a.py", child("child_a", 10.0))
|
||||
self.write("child_b.py", child("child_b", 6.0))
|
||||
self.top = self.write("top.py", self.parent_text())
|
||||
|
||||
@staticmethod
|
||||
def parent_text() -> str:
|
||||
# both uses in one cell, as in a real assembly
|
||||
return notebook_text(
|
||||
"@app.cell\ndef _():\n a = scad.use('child_a.py')\n b = scad.use('child_b.py')\n return a, b\n",
|
||||
"@app.cell\ndef _(a, b):\n" + ASSEMBLE.rstrip("\n") + "\n return (top,)\n",
|
||||
table='id = "top"',
|
||||
)
|
||||
|
||||
def test_a_cell_with_two_uses_starts_them_together_and_gives_the_serial_result(self) -> None:
|
||||
run = run_notebook(self.top)
|
||||
self.assertEqual(sorted(Path(p).name for p in prefetch.stats()["submitted"]), ["child_a.py", "child_b.py"])
|
||||
serial = run_notebook(self.top, cache=False)
|
||||
self.assertEqual(run.definition.content_hash, serial.definition.content_hash)
|
||||
|
||||
def test_a_notebook_that_has_not_changed_since_it_last_ran_is_not_started_again(self) -> None:
|
||||
run_notebook(self.top)
|
||||
prefetch.reset_stats()
|
||||
run_notebook(self.top, cache=False) # the cell runs again; the children are as they were
|
||||
self.assertEqual(prefetch.stats()["submitted"], [])
|
||||
|
||||
def stamped(self, seconds: float) -> None:
|
||||
"""Make both children look like they last took *seconds* to build."""
|
||||
|
||||
from simplecadapi.runtime.config import read_notebook_config
|
||||
|
||||
for name in ("child_a.py", "child_b.py"):
|
||||
stamp = prefetch._stamp_file(read_notebook_config(self.root / name))
|
||||
stamp.parent.mkdir(parents=True, exist_ok=True)
|
||||
stamp.write_text(json.dumps({"signature": "stale", "seconds": seconds}))
|
||||
|
||||
def test_notebooks_that_build_in_moments_are_not_worth_a_worker(self) -> None:
|
||||
# a worker takes seconds to start; two builds of a fifth of a second do not repay it
|
||||
self.stamped(0.2)
|
||||
run_notebook(self.top)
|
||||
self.assertEqual(prefetch.stats()["submitted"], [])
|
||||
|
||||
def test_notebooks_that_took_long_are_started_together(self) -> None:
|
||||
self.stamped(20.0)
|
||||
run_notebook(self.top)
|
||||
self.assertEqual(len(prefetch.stats()["submitted"]), 2)
|
||||
|
||||
def test_a_run_remembers_how_long_it_took_to_build(self) -> None:
|
||||
from simplecadapi.runtime.config import read_notebook_config
|
||||
|
||||
run_notebook(self.root / "child_a.py")
|
||||
stamp = json.loads(prefetch._stamp_file(read_notebook_config(self.root / "child_a.py")).read_text())
|
||||
self.assertGreater(stamp["seconds"], 0.0)
|
||||
# a run that only restores does not make it look cheap
|
||||
run_notebook(self.root / "child_a.py", cache=True)
|
||||
again = json.loads(prefetch._stamp_file(read_notebook_config(self.root / "child_a.py")).read_text())
|
||||
self.assertEqual(again["seconds"], stamp["seconds"])
|
||||
|
||||
def test_a_failing_child_is_reported_by_the_run_that_uses_it(self) -> None:
|
||||
self.write("child_b.py", child("child_b", 6.0).replace("make_box_rsolid(width, 4.0, 4.0)", "make_box_rsolid(width, -4.0, 4.0)"))
|
||||
with self.assertRaises(Exception) as caught:
|
||||
run_notebook(self.top)
|
||||
# the error of the child itself, not something about a worker
|
||||
self.assertIn("greater than zero", str(caught.exception))
|
||||
|
||||
def test_one_use_is_not_worth_a_worker(self) -> None:
|
||||
one = notebook_text(
|
||||
"@app.cell\ndef _():\n a = scad.use('child_a.py')\n return (a,)\n",
|
||||
"@app.cell\ndef _(a):\n"
|
||||
' _asm = scad.make_assembly_rassembly(assembly_id="single", name="single")\n'
|
||||
' single = scad.add_component_rassembly(assembly=_asm, item=a, component_id="a", '
|
||||
"placement=scad.identity_placement_rplacement())\n"
|
||||
" return (single,)\n",
|
||||
table='id = "single"',
|
||||
)
|
||||
run_notebook(self.write("single.py", one))
|
||||
self.assertEqual(prefetch.stats()["submitted"], [])
|
||||
|
||||
def test_the_command_line_prints_only_its_report(self) -> None:
|
||||
chatty = child("child_a", 10.0).replace("width = 10.0", "print('from the child')\n width = 10.0")
|
||||
self.write("child_a.py", chatty)
|
||||
done = subprocess.run(
|
||||
[sys.executable, "-c", "import sys; from simplecadapi.cli import main; sys.exit(main(['run', sys.argv[1]]))", str(self.top)],
|
||||
capture_output=True, text=True, env={**os.environ, "SCA_JOBS": "2"}, timeout=300,
|
||||
)
|
||||
self.assertEqual(done.returncode, 0, done.stderr[-2000:])
|
||||
lines = [line for line in done.stdout.splitlines() if line.strip()]
|
||||
self.assertEqual(json.loads(lines[-1])["definition_id"], "top")
|
||||
self.assertNotIn("from the child", done.stdout) # a worker's output goes nowhere
|
||||
Reference in New Issue
Block a user