perf(runtime): static use digests, process-level use prefetch, state-dir lock

A cell that uses notebooks by literal path is keyed by what each of them is,
and the kernel caches such cells; the use calls a cell is about to make are
started together in worker processes (SCA_JOBS), when that is worth the
seconds a worker takes to start; a notebook's state directory is locked
between processes. Desktop-arm benchmark: cold 80.5s -> 34.6s.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
NiJingzhe
2026-10-21 00:25:02 +08:00
parent 54016e9a97
commit df52326ec0
12 changed files with 1044 additions and 7 deletions
+14
View File
@@ -23,6 +23,20 @@ user-facing notes live in [`docs/updates/`](docs/updates/) (English and
cell, changed (marimo keyed any object it cannot hash by the code of the cell that made it).
- Editor: a slider drag is sent when it pauses (250 ms) or ends, not step by step; clicks and
number-box commits still go at once.
- Cache: a cell that runs a notebook with `use("child.py")` (a literal path) is keyed by what
that child is (its text, the modules it imports, the notebooks it uses in turn) instead of
by everything a run recorded: editing one child redoes the cell that uses it and not those
that use its siblings, and the editor kernel caches such cells too (it ran them live
before). A path computed at run time still depends on what a run recorded.
- Runtime: the `use` calls a cell is about to make (literal path; arguments made of values it
reads, `float(...)`, `notebook_id()`) are started together in worker processes, which fill
the notebooks' caches; the cell then finds them built. OpenCASCADE keeps the interpreter
lock, so these are processes, not threads. `SCA_JOBS` sets how many (default: the smaller of
4 and the cores; 1 turns it off). Workers are only started when the builds they take off
the cell's hands are worth the seconds a worker takes to start (a notebook remembers how
long it last took to build), and a notebook that has not changed since it last ran is not
started at all. A notebook's state directory is locked between processes while it runs.
Desktop-arm benchmark: cold 80.5 s to 34.6 s.
- Cache: a cell that does `from lib.dims import A` now depends on the definition of `A` (and
what it reads, across local modules), not on the whole file. Editing another name of
`lib/dims.py`, a comment, or the formatting leaves the cell cached. A module the analysis
@@ -0,0 +1,41 @@
"""A worker process of :mod:`.prefetch`: runs the notebooks it is sent.
Jobs and answers are length-prefixed pickles on stdin and the original stdout;
file descriptor 1 itself is pointed at the null device, so whatever a notebook
prints (from Python or from C) cannot get in the way of the protocol.
"""
from __future__ import annotations
import os
import pickle
import struct
import sys
from pathlib import Path
def main() -> int:
reply = os.fdopen(os.dup(1), "wb")
null = os.open(os.devnull, os.O_WRONLY)
os.dup2(null, 1)
sys.stdout = open(os.devnull, "w") # noqa: SIM115
jobs = sys.stdin.buffer
from simplecadapi.runtime.runner import run_notebook
while True:
header = jobs.read(8)
if len(header) < 8:
return 0
path, child_id, overrides = pickle.loads(jobs.read(struct.unpack(">Q", header)[0]))
try:
run_notebook(Path(path), id=child_id, overrides=overrides)
answer: tuple[str, str] = ("ok", "")
except BaseException as error: # noqa: BLE001 - reported, the parent runs it for itself
answer = ("error", f"{type(error).__name__}: {error}"[-2000:])
data = pickle.dumps(answer)
reply.write(struct.pack(">Q", len(data)) + data)
reply.flush()
if __name__ == "__main__":
sys.exit(main())
+27 -3
View File
@@ -41,6 +41,7 @@ from ._marimo import (
RunResult,
Skip,
)
from .children import notebook_digest
from .config import NotebookConfig
from .names import names_tag
from .stubs import register_scalar_stubs # noqa: F401 (registers on import)
@@ -50,7 +51,6 @@ from .deps import (
cell_digest,
cell_module_files,
read_dependencies,
recorded_dependents,
state_dir,
)
@@ -133,7 +133,8 @@ class CellDigests:
# A cell that runs a child notebook, or reads what one makes, depends
# on what the last run recorded; its entries are kept apart so they
# can follow the record.
self._uses = recorded_dependents(sources)
self._uses = frozenset(key for key, deps in self._files.items() if deps.dynamic)
self._recorded = read_dependencies(config)
self._kernel = kernel
self._saved_tag: dict[str, str] = {}
if saved:
@@ -154,9 +155,32 @@ class CellDigests:
if self._kernel and cell_id in self._uses:
return None
named, fallback = names_tag(self.config.directory, deps.names)
tag = ("use" if cell_id in self._uses else "") + self._saved_tag.get(cell_id, "") + named
used, unknown = self._used_tag(deps.uses)
if unknown:
# A notebook this one uses has a part of its own that only a run
# can name: depend on what the last run recorded.
if self._kernel:
return None
fallback = fallback | self._recorded
tag = ("use" if cell_id in self._uses or unknown else "") + self._saved_tag.get(cell_id, "") + named + used
return cell_digest(self.config, deps.files | fallback, tag=tag)
@staticmethod
def _used_tag(paths: frozenset[Path]) -> tuple[str, bool]:
"""The digest of the notebooks a cell uses by a literal path, and
whether one of them cannot be digested."""
if not paths:
return "", False
digests: dict[str, str] = {}
for path in sorted(paths):
digest = notebook_digest(path)
if digest is None:
return "", True
digests[path.name] = digest
encoded = json.dumps(digests, sort_keys=True).encode("utf-8")
return "u" + hashlib.sha256(encoded).hexdigest()[:12], False
class CellCache(CachedLifecycle):
"""marimo's cell cache, a store per digest, reporting its hits."""
+115
View File
@@ -0,0 +1,115 @@
"""What a notebook is, to the notebook that uses it.
``use("child.py")`` makes a cell depend on the child. When the path is a
literal that is known before the cell runs, so the cell is keyed by a digest
of the child, and each ``use`` is followed on its own: editing one child
recomputes the cells that use it, not those that use its siblings.
The digest is of what the child *is*: its text, the local modules it imports,
the project files it reads, and, in turn, the digest of every notebook it
uses. Its text, not its syntax tree: the definition a run makes records where
in the file each step is, so a comment or a blank line changes the definition
of every notebook that embeds the child, and a cached parent must equal a
fresh one.
A child that computes a path itself has no such digest (``None``): the caller
then depends on whatever a run recorded.
"""
from __future__ import annotations
import hashlib
import json
import os
import threading
from dataclasses import dataclass
from pathlib import Path
from .config import read_notebook_config
from .deps import file_digest, input_files, local_modules, use_calls
@dataclass(frozen=True)
class _Parts:
syntax: str
header: str
modules: tuple[Path, ...]
inputs: tuple[Path, ...]
children: tuple[Path, ...]
_parts_memo: dict[tuple[Path, str | None], _Parts | None] = {}
_memo_lock = threading.Lock()
def notebook_digest(path: Path) -> str | None:
"""The digest of the notebook at *path* and what it depends on, or ``None``
when that cannot be told without running it."""
return _digest(path.resolve(), frozenset())
def _digest(path: Path, stack: frozenset[Path]) -> str | None:
if path in stack:
return "cycle" # a run reports it
parts = _parts(path)
if parts is None:
return None
here = path.parent
children: dict[str, str] = {}
for child in parts.children:
digest = _digest(child, stack | {path})
if digest is None:
return None
children[_relative(child, here)] = digest
payload = {
"syntax": parts.syntax,
"header": parts.header,
"modules": {_relative(item, here): file_digest(item) for item in parts.modules},
"inputs": {_relative(item, here): file_digest(item) for item in parts.inputs},
"children": children,
}
encoded = json.dumps(payload, sort_keys=True).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
def _parts(path: Path) -> _Parts | None:
key = (path, file_digest(path))
with _memo_lock:
if key in _parts_memo:
return _parts_memo[key]
parts = _analyse(path)
with _memo_lock:
if len(_parts_memo) > 512:
_parts_memo.clear()
_parts_memo[key] = parts
return parts
def _analyse(path: Path) -> _Parts | None:
try:
text = path.read_text(encoding="utf-8")
except (OSError, UnicodeError):
return None
config = read_notebook_config(path)
if config is None:
return None
calls = use_calls(text)
if any(call is None for call in calls):
return None
return _Parts(
syntax=hashlib.sha256(text.encode("utf-8")).hexdigest()[:16],
header=repr((config.id, config.revision, config.tolerance_profile, config.inputs)),
modules=tuple(sorted(local_modules(config, [text]).values())),
inputs=tuple(sorted(input_files(config))),
children=tuple(sorted({(path.parent / call).resolve() for call in calls if call is not None})),
)
def _relative(path: Path, start: Path) -> str:
try:
return Path(os.path.relpath(path, start)).as_posix()
except ValueError: # another drive on Windows
return path.as_posix()
__all__ = ["notebook_digest"]
+83 -3
View File
@@ -92,6 +92,49 @@ def state_dir(config: NotebookConfig) -> Path:
return notebook_output_dir(config.path) / "simplecad" / config.path.name / config.id
_held: set[str] = set()
_held_lock = threading.Lock()
@contextmanager
def exclusive(config: NotebookConfig) -> Iterator[None]:
"""Hold the notebook's state directory against other processes.
Worker processes may run the same notebook at once (two notebooks that
use a third), and a run prunes the cache directories it did not use. The
lock is between processes only: a run of a notebook this process is
already running (a ``use`` cycle, an editor next to a headless run) goes
on without it, as it always did. Without ``fcntl`` (Windows) it is a
no-op.
"""
directory = state_dir(config)
key = str(directory)
with _held_lock:
nested = key in _held
_held.add(key)
handle = None
try:
if not nested:
try:
import fcntl
directory.mkdir(parents=True, exist_ok=True)
handle = open(directory / "run.lock", "a") # noqa: SIM115
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
except (ImportError, OSError):
if handle is not None:
handle.close()
handle = None
yield
finally:
if handle is not None:
handle.close() # releases the lock
if not nested:
with _held_lock:
_held.discard(key)
def deps_file(config: NotebookConfig) -> Path:
return state_dir(config) / "deps.json"
@@ -312,9 +355,18 @@ class CellDeps:
files: frozenset[Path]
names: frozenset[NameDep] = frozenset()
uses: frozenset[Path] = frozenset()
"""Notebooks the cell runs with ``use``, by a literal path (see :mod:`.children`)."""
dynamic: bool = False
"""The cell runs a notebook whose path is computed: only a run knows which."""
def __or__(self, other: CellDeps) -> CellDeps:
return CellDeps(self.files | other.files, self.names | other.names)
return CellDeps(
self.files | other.files,
self.names | other.names,
self.uses | other.uses,
self.dynamic or other.dynamic,
)
def cell_module_files(
@@ -411,9 +463,14 @@ def cell_module_files(
named.add(NameDep(module_name, original, frozenset(coarse)))
else:
files |= coarse
if calls_use(cell.code):
calls = use_calls(cell.code)
dynamic = any(call is None for call in calls)
if dynamic:
files |= recorded_files
result[cell.key] = CellDeps(frozenset(files), frozenset(named))
uses = frozenset(
(root / call).resolve() for call in calls if call is not None
)
result[cell.key] = CellDeps(frozenset(files), frozenset(named), uses, dynamic)
# marimo keys a cell that reads an object (a solid, a part) by the code of
# the cells that made it, not by its value: a file an ancestor depends on
@@ -565,6 +622,27 @@ def _loaded_names_of(node: ast.AST) -> set[str]:
}
def use_calls(source: str) -> list[str | None]:
"""The paths of the ``use(...)`` calls in *source*: the literal, or ``None``
where it is computed (or the source cannot be read)."""
try:
tree = ast.parse(source)
except (SyntaxError, ValueError):
return [None]
paths: list[str | None] = []
for node in ast.walk(tree):
if isinstance(node, ast.Call):
function = node.func
if (isinstance(function, ast.Name) and function.id == "use") or (
isinstance(function, ast.Attribute) and function.attr == "use"
):
first = node.args[0] if node.args else None
literal = isinstance(first, ast.Constant) and isinstance(first.value, str)
paths.append(first.value if literal else None) # type: ignore[union-attr]
return paths
def calls_use(source: str) -> bool:
try:
tree = ast.parse(source)
@@ -668,12 +746,14 @@ __all__ = [
"NameDep",
"ancestors",
"calls_use",
"use_calls",
"cell_digest",
"cell_module_files",
"dependency_digest",
"file_digest",
"recorded_dependents",
"deps_file",
"exclusive",
"input_files",
"local_modules",
"read_dependencies",
+12
View File
@@ -45,6 +45,7 @@ from ._marimo import (
RunResult,
safe_get_context,
)
from . import prefetch
from .cache import CellCache, CellDigests, cache_root, prune
from .cells import NotebookCells, bound_requirements
from .config import NotebookConfig, read_notebook_config
@@ -116,6 +117,7 @@ class SimpleCadExecutor:
if restored is not _MISS:
return restored
cache = cache if restored is _MISS else None
_prefetch(scope, cell, glbls)
try:
with _cell_recording(scope, cell, glbls):
result = self._delegate.execute_cell(cell, glbls)
@@ -134,6 +136,7 @@ class SimpleCadExecutor:
restored = _restore(cache, cell, glbls)
if restored is not _MISS:
return restored
_prefetch(scope, cell, glbls)
try:
with _cell_recording(scope, cell, glbls):
result = await self._delegate.execute_cell_async(cell, glbls)
@@ -160,6 +163,15 @@ class SimpleCadExecutor:
return _kernel_cache(scope, cell)
def _prefetch(scope: NotebookScope, cell: CellImpl, glbls: MutableGlobals) -> None:
"""Start the notebooks the cell is about to ``use`` in worker processes."""
try:
prefetch.start(scope.config.directory, cell.code, glbls, scope.config.id)
except Exception: # noqa: BLE001 - an optimisation, never a reason for a cell to fail
pass
_MISS = object()
"""Returned by :func:`_restore` when the cell must run."""
+418
View File
@@ -0,0 +1,418 @@
"""Run the notebooks a cell uses side by side.
OpenCASCADE does not let go of the interpreter lock (two threads building
the same kind of model are no faster than one), so the parallelism is in
processes. When a cell is about to run, the ``use`` calls it is going to make
that can be told beforehand (:func:`plan_uses`: literal paths, arguments made
of values the cell reads) are handed to worker processes. A worker runs the
notebook headlessly, which fills its cell cache; the cell then makes its
``use`` calls as it always did, waits for the worker of each (:func:`wait`),
and finds the notebook built. Nothing about the result depends on the
workers: a failed or missing one only means the call runs the notebook itself.
``SCA_JOBS`` sets how many workers (default: the smaller of 4 and the number
of cores; 1 turns this off). A notebook that has not changed since it last
ran, with the same overrides, is not started: it is in the cache already.
"""
from __future__ import annotations
import ast
import hashlib
import json
import os
import pickle
import struct
import subprocess
import sys
import threading
from collections.abc import Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any
from ..artifacts.canonical import validate_logical_id
from .children import notebook_digest
from .config import read_notebook_config
from .deps import state_dir, use_calls
# ---------------------------------------------------------------------------
# What a cell is going to use
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class UseRequest:
path: Path
id: str | None
overrides: Mapping[str, Any]
@property
def key(self) -> tuple[str, str | None, str]:
return request_key(self.path, self.id, self.overrides)
def request_key(path: Path, id: str | None, overrides: Mapping[str, Any]) -> tuple[str, str | None, str]:
return (str(Path(path).resolve()), id, repr(sorted(overrides.items(), key=lambda item: item[0])))
class _Unknown(Exception):
"""An expression the plan cannot evaluate before the cell runs."""
_CALLS = {"float": float, "int": int, "str": str, "bool": bool, "round": round, "abs": abs, "min": min, "max": max}
_BINARY = {
ast.Add: lambda a, b: a + b,
ast.Sub: lambda a, b: a - b,
ast.Mult: lambda a, b: a * b,
ast.Div: lambda a, b: a / b,
ast.FloorDiv: lambda a, b: a // b,
ast.Mod: lambda a, b: a % b,
ast.Pow: lambda a, b: a**b,
}
_COMPARE = {
ast.Eq: lambda a, b: a == b,
ast.NotEq: lambda a, b: a != b,
ast.Lt: lambda a, b: a < b,
ast.LtE: lambda a, b: a <= b,
ast.Gt: lambda a, b: a > b,
ast.GtE: lambda a, b: a >= b,
}
_NOT_RUN_HERE = (
ast.IfExp, ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.FunctionDef,
ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda, ast.ListComp, ast.SetComp,
ast.DictComp, ast.GeneratorExp, ast.Match,
) # fmt: skip
def _evaluate(node: ast.expr, scope: Mapping[str, Any], notebook_id: str | None = None) -> Any:
"""The value of *node*, for the few expression forms a plan can trust."""
if isinstance(node, ast.Constant):
return node.value
if (
isinstance(node, ast.Call)
and not node.args
and not node.keywords
and (
(isinstance(node.func, ast.Name) and node.func.id == "notebook_id")
or (isinstance(node.func, ast.Attribute) and node.func.attr == "notebook_id")
)
):
if notebook_id is None:
raise _Unknown
return notebook_id
if isinstance(node, ast.Name):
if node.id in scope:
return scope[node.id]
raise _Unknown
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd, ast.Not)):
value = _evaluate(node.operand, scope, notebook_id)
return {ast.USub: lambda: -value, ast.UAdd: lambda: +value, ast.Not: lambda: not value}[type(node.op)]()
if isinstance(node, ast.BinOp) and type(node.op) in _BINARY:
return _BINARY[type(node.op)](_evaluate(node.left, scope, notebook_id), _evaluate(node.right, scope, notebook_id))
if isinstance(node, ast.Compare) and len(node.ops) == 1 and type(node.ops[0]) in _COMPARE:
return _COMPARE[type(node.ops[0])](_evaluate(node.left, scope, notebook_id), _evaluate(node.comparators[0], scope, notebook_id))
if isinstance(node, ast.IfExp):
return _evaluate(node.body if _evaluate(node.test, scope, notebook_id) else node.orelse, scope, notebook_id)
if isinstance(node, (ast.Tuple, ast.List)):
items = [_evaluate(item, scope, notebook_id) for item in node.elts]
return tuple(items) if isinstance(node, ast.Tuple) else items
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id in _CALLS
and node.func.id not in scope
and not node.keywords
):
return _CALLS[node.func.id](*[_evaluate(arg, scope, notebook_id) for arg in node.args])
raise _Unknown
def plan_uses(
code: str, scope: Mapping[str, Any], directory: Path, notebook_id: str | None = None
) -> list[UseRequest]:
"""The ``use`` calls *code* will make that are known before it runs.
A call counts when its path is a literal and every argument can be worked
out from *scope* (the values the cell reads) with arithmetic and
``float(...)``-like calls; and when the cell is certain to make it (not
under a condition, in a loop or in a function).
"""
try:
tree = ast.parse(code)
except (SyntaxError, ValueError):
return []
requests: dict[tuple[str, str | None, str], UseRequest] = {}
def visit(node: ast.AST) -> None:
if isinstance(node, _NOT_RUN_HERE):
return
if isinstance(node, ast.Call) and _is_use(node):
request = _request(node, scope, directory, notebook_id)
if request is not None:
requests.setdefault(request.key, request)
for child in ast.iter_child_nodes(node):
visit(child)
visit(tree)
return list(requests.values())
def _is_use(node: ast.Call) -> bool:
function = node.func
return (isinstance(function, ast.Name) and function.id == "use") or (
isinstance(function, ast.Attribute) and function.attr == "use"
)
def _request(
node: ast.Call, scope: Mapping[str, Any], directory: Path, notebook_id: str | None
) -> UseRequest | None:
if not node.args or not (isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str)):
return None
if len(node.args) != 1 or any(keyword.arg is None for keyword in node.keywords):
return None
try:
values = {keyword.arg: _evaluate(keyword.value, scope, notebook_id) for keyword in node.keywords}
except Exception: # noqa: BLE001 - anything wrong here only means "not planned"
return None
child_id = values.pop("id", None)
if child_id is not None and not isinstance(child_id, str):
return None
path = Path(node.args[0].value)
return UseRequest((path if path.is_absolute() else directory / path).resolve(), child_id, values)
# ---------------------------------------------------------------------------
# Has the notebook changed since it last ran?
# ---------------------------------------------------------------------------
def _signature(path: Path, config_id: str, overrides: Mapping[str, Any]) -> str | None:
digest = notebook_digest(path)
if digest is None:
return None
payload = repr((digest, config_id, sorted(overrides.items(), key=lambda item: item[0])))
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:20]
def _stamp_file(config: Any) -> Path:
return state_dir(config) / "stamp.json"
def _read_stamp(config: Any) -> dict[str, Any]:
try:
data = json.loads(_stamp_file(config).read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
return data if isinstance(data, dict) else {}
def write_stamp(config: Any, overrides: Mapping[str, Any], seconds: float | None = None) -> None:
"""Record that *config*'s notebook has just run to the end with *overrides*.
*seconds* is how long the run took when it built something; a run that
only restored keeps the figure of the last build (it says what the
notebook costs, which is what decides whether a worker is worth it).
"""
signature = _signature(config.path, config.id, overrides)
target = _stamp_file(config)
try:
if signature is None:
target.unlink(missing_ok=True)
return
previous = _read_stamp(config)
if seconds is None:
seconds = previous.get("seconds")
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps({"signature": signature, "seconds": seconds}), encoding="utf-8")
except OSError:
pass # the stamp only saves work
def _expected_seconds(request: UseRequest) -> float | None:
"""How long the notebook has to be built, if it has to be: ``None`` when it
does not (it is in the cache), ``inf`` when nobody knows how long."""
config = read_notebook_config(request.path)
if config is None:
return None # the call itself will say what is wrong with it
try:
if request.id is not None:
config = replace(config, id=validate_logical_id(request.id, "/run/id"))
except Exception: # noqa: BLE001
return None
signature = _signature(config.path, config.id, request.overrides)
stamp = _read_stamp(config)
if signature is not None and stamp.get("signature") == signature:
return None
seconds = stamp.get("seconds")
return float(seconds) if isinstance(seconds, (int, float)) else float("inf")
# ---------------------------------------------------------------------------
# Workers
# ---------------------------------------------------------------------------
_COLD_WORTH = 4.0
"""Seconds a start of the workers has to save to be done (a worker imports the
kernel libraries: a few seconds)."""
_WARM_WORTH = 0.5
"""...and once they are running."""
_stats: dict[str, list[str]] = {"submitted": []}
_pending: dict[tuple[str, str | None, str], Future[None]] = {}
_lock = threading.Lock()
_pool: _Pool | None = None
def jobs() -> int:
raw = os.environ.get("SCA_JOBS")
if raw is not None:
try:
return max(1, int(raw))
except ValueError:
return 1
return max(1, min(4, os.cpu_count() or 1))
def stats() -> dict[str, list[str]]:
return {"submitted": list(_stats["submitted"])}
def reset_stats() -> None:
_stats["submitted"].clear()
def start(directory: Path, code: str, scope: Mapping[str, Any], notebook_id: str | None = None) -> None:
"""Hand the ``use`` calls of a cell about to run to the workers."""
global _pool
size = jobs()
if size < 2 or "use" not in code:
return
if not use_calls(code):
return
planned = [(request, _expected_seconds(request)) for request in plan_uses(code, scope, directory, notebook_id)]
wanted = [(request, seconds) for request, seconds in planned if seconds is not None]
if len(wanted) < 2:
return # nothing to do beside the cell's own first call
# The longest build goes on in this process; what the workers take off its
# hands has to be worth more than starting them.
seconds = sorted((value for _, value in wanted), reverse=True)
saved = sum(seconds[1:])
if saved < (_WARM_WORTH if _pool is not None and _pool.size == size else _COLD_WORTH):
return
requests = [request for request, _ in wanted]
with _lock:
if _pool is None or _pool.size != size:
_pool = _Pool(size)
for request in requests:
existing = _pending.get(request.key)
if existing is not None and not existing.done():
continue
_pending[request.key] = _pool.submit(request)
_stats["submitted"].append(str(request.path))
def wait(path: Path, id: str | None, overrides: Mapping[str, Any]) -> None:
"""Wait for the worker that is building this notebook, if one is."""
with _lock:
future = _pending.pop(request_key(path, id, overrides), None)
if future is not None:
try:
future.result()
except Exception: # noqa: BLE001 - the call runs the notebook and reports for itself
pass
def shutdown() -> None:
global _pool
with _lock:
pool, _pool = _pool, None
_pending.clear()
if pool is not None:
pool.close()
class _Pool:
"""Threads that each own a worker process, started when first needed."""
def __init__(self, size: int) -> None:
self.size = size
self._executor = ThreadPoolExecutor(size, thread_name_prefix="sca-prefetch")
self._local = threading.local()
self._workers: list[subprocess.Popen[bytes]] = []
self._workers_lock = threading.Lock()
def submit(self, request: UseRequest) -> Future[None]:
return self._executor.submit(self._run, request)
def _run(self, request: UseRequest) -> None:
worker = getattr(self._local, "worker", None)
if worker is None or worker.poll() is not None:
worker = self._local.worker = self._spawn()
try:
_send(worker.stdin, (str(request.path), request.id, dict(request.overrides))) # type: ignore[arg-type]
reply = _receive(worker.stdout) # type: ignore[arg-type]
except Exception:
self._local.worker = None
worker.kill()
raise
if reply[0] != "ok":
raise RuntimeError(reply[1])
def _spawn(self) -> subprocess.Popen[bytes]:
worker = subprocess.Popen(
[sys.executable, "-m", "simplecadapi.runtime._prefetch_worker"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
env={**os.environ, "SCA_JOBS": "1"}, # a worker does not start workers
)
with self._workers_lock:
self._workers.append(worker)
return worker
def close(self) -> None:
self._executor.shutdown(wait=False, cancel_futures=True)
with self._workers_lock:
workers, self._workers = self._workers, []
for worker in workers:
try:
worker.kill()
except OSError:
pass
def _send(stream: Any, value: Any) -> None:
data = pickle.dumps(value)
stream.write(struct.pack(">Q", len(data)) + data)
stream.flush()
def _receive(stream: Any) -> Any:
header = stream.read(8)
if len(header) < 8:
raise EOFError("the worker ended")
(size,) = struct.unpack(">Q", header)
return pickle.loads(stream.read(size))
__all__ = [
"UseRequest",
"jobs",
"plan_uses",
"request_key",
"reset_stats",
"shutdown",
"start",
"stats",
"wait",
"write_stamp",
]
+10 -1
View File
@@ -24,6 +24,7 @@ from __future__ import annotations
import hashlib
import sys
import time
from collections.abc import Callable, Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
@@ -53,11 +54,13 @@ from ._marimo import (
safe_get_context,
suppress_warnings_logs,
)
from . import prefetch
from .cache import CellCache, CellDigests, cache_root, prune
from .cells import NotebookCells
from .config import NotebookConfig, NotebookConfigError, read_notebook_config
from .deps import (
local_modules,
exclusive,
read_dependencies,
record_dependencies,
recording_dependencies,
@@ -163,7 +166,7 @@ def run_notebook(
# Errors still come through. "marimo" is the logger every marimo
# module shares (lazy.py's record shows its file name, "lazy", which
# is not the logger's name).
with suppress_warnings_logs("marimo"):
with suppress_warnings_logs("marimo"), exclusive(config):
return _run(config, values, cache=cache, through=through, product=product)
# marimo keeps its runtime context per thread, and a notebook run needs
@@ -188,6 +191,7 @@ def use(path: str | Path, /, *, id: str | None = None, **overrides: Any) -> Prod
target = Path(path)
if not target.is_absolute():
target = _calling_directory() / target
prefetch.wait(target, id, overrides) # a worker may be building it
child = run_notebook(target, id=id, overrides=overrides)
# use() never asks for a partial run, so the product is always there.
if child.product is None:
@@ -209,6 +213,8 @@ def _run(
through: int | None = None,
product: bool = True,
) -> NotebookRun:
started = time.perf_counter()
given = dict(overrides) # (``overrides`` becomes the run's values below)
app = load_notebook(config.path, cell_prefix=_cell_prefix(config))
if app is None:
raise NotebookConfigError(f"{config.path}: the notebook has no cells")
@@ -310,6 +316,9 @@ def _run(
)
for cell in ordered
)
if product and through is None and cache:
built = any(report.status == "ran" and report.name != "setup" for report in reports)
prefetch.write_stamp(config, given, time.perf_counter() - started if built else None)
return NotebookRun(
config=config,
product=product_value,
+5
View File
@@ -11,3 +11,8 @@ import os
# with the agent-facing default.
os.environ.setdefault("SCA_NO_DIAGNOSTIC_RENDER", "1")
os.environ.setdefault("SCA_DIAGNOSTIC_RENDER_BUDGET", "60")
# Worker processes for the notebooks a cell uses are for the tests that are
# about them (they set SCA_JOBS themselves): starting several interpreters per
# run would only slow the rest of the suite down.
os.environ.setdefault("SCA_JOBS", "1")
+51
View File
@@ -0,0 +1,51 @@
"""One process at a time in a notebook's state directory.
Workers (see ``prefetch``) may run the same notebook at once, and a run
prunes the cache directories it did not use. The lock is between processes:
inside one, a nested run of the same notebook (a ``use`` cycle) goes on as it
always did, and is reported, not waited for.
"""
from __future__ import annotations
import subprocess
import sys
import textwrap
import time
from simplecadapi.runtime.config import read_notebook_config
from simplecadapi.runtime.deps import exclusive
from test_notebook_runtime import NotebookTestCase, cell, notebook_text
class TestStateLock(NotebookTestCase):
def setUp(self) -> None:
super().setUp()
self.path = self.write("n.py", notebook_text(cell("x = 1", returns="x")))
self.config = read_notebook_config(self.path)
def test_a_nested_run_of_the_same_notebook_is_not_waited_for(self) -> None:
with exclusive(self.config):
with exclusive(self.config):
pass
def test_another_process_waits_for_the_holder(self) -> None:
holder = textwrap.dedent(
f"""
import sys, time
from simplecadapi.runtime.config import read_notebook_config
from simplecadapi.runtime.deps import exclusive
with exclusive(read_notebook_config({str(self.path)!r})):
print("locked", flush=True)
time.sleep(1.5)
"""
)
process = subprocess.Popen([sys.executable, "-c", holder], stdout=subprocess.PIPE, text=True)
try:
self.assertEqual(process.stdout.readline().strip(), "locked")
started = time.monotonic()
with exclusive(self.config):
waited = time.monotonic() - started
self.assertGreater(waited, 0.8)
finally:
process.wait(timeout=30)
+110
View File
@@ -0,0 +1,110 @@
"""`use` in the cache: a cell depends on the notebooks it uses, one by one.
The path of a ``use`` call is usually a literal, so which notebooks a cell
depends on is known before it runs: the cell is keyed by what each of them is
(its code, not its comments or layout; the modules it imports; the notebooks
it uses in turn). What is computed at run time is recorded by a run, as
before.
"""
from __future__ import annotations
from simplecadapi.runtime import run_notebook
from simplecadapi.runtime._marimo import InternalApp, load_notebook
from simplecadapi.runtime.cache import CellDigests
from simplecadapi.runtime.config import read_notebook_config
from simplecadapi.runtime.runner import _cell_prefix
from test_notebook_runtime import NotebookTestCase, cell, notebook_text
def child(name: str, width: float, note: str = "") -> str:
return notebook_text(
cell(f"width = {width}{note}", returns="width"),
cell("block = scad.make_box_rsolid(width, 4.0, 4.0)", reads="width", returns="block"),
cell(f"part = scad.Part(part_id='{name}', body=block)", reads="block", returns="part"),
table=f'id = "{name}"',
)
ASSEMBLE = """
_asm = scad.make_assembly_rassembly(assembly_id="top", name="top")
for _key, _item in (("a", a), ("b", b)):
_asm = scad.add_component_rassembly(
assembly=_asm, item=_item, component_id=_key, placement=scad.identity_placement_rplacement()
)
top = _asm
"""
def parent(first: str = 'scad.use("child_a.py")', second: str = 'scad.use("child_b.py")', *, before: str = "") -> str:
cells = [cell(f"{before}\na = {first}" if before else f"a = {first}", returns="a")]
cells += [cell(f"b = {second}", returns="b"), cell(ASSEMBLE, reads="a, b", returns="top")]
return notebook_text(*cells, table='id = "top"')
class TestStaticUse(NotebookTestCase):
def setUp(self) -> None:
super().setUp()
self.write("child_a.py", child("child_a", 10.0))
self.write("child_b.py", child("child_b", 6.0))
self.top = self.write("top.py", parent())
def edited(self) -> list[str]:
run = run_notebook(self.top)
fresh = run_notebook(self.top, cache=False)
self.assertEqual(run.definition.content_hash, fresh.definition.content_hash)
return self.statuses(run)
def test_editing_one_child_redoes_only_the_cell_that_uses_it(self) -> None:
run_notebook(self.top)
self.write("child_a.py", child("child_a", 12.0))
# setup, use a, use b, the assembly
self.assertEqual(self.edited(), ["ran", "ran", "cached", "ran"])
def test_a_comment_in_a_child_is_a_change_like_any_other(self) -> None:
# The definition records where in the file each step is: a parent that
# embeds the child must equal a fresh run, comment or not.
run_notebook(self.top)
self.write("child_a.py", child("child_a", 10.0, note=" # the width, in mm"))
self.assertEqual(self.edited(), ["ran", "ran", "cached", "ran"])
def test_a_module_a_child_imports_is_followed(self) -> None:
self.write("lib/size.py", "SIZE = 5.0\n")
text = child("child_a", 10.0).replace(
"with app.setup:\n import simplecadapi as scad\n",
"with app.setup:\n import simplecadapi as scad\n from lib.size import SIZE\n",
).replace("make_box_rsolid(width, 4.0, 4.0)", "make_box_rsolid(width, SIZE, 4.0)")
self.write("child_a.py", text)
run_notebook(self.top)
self.write("lib/size.py", "SIZE = 6.0\n")
self.assertEqual(self.edited(), ["ran", "ran", "cached", "ran"])
def test_a_grandchild_is_followed_through_the_child(self) -> None:
self.write("deep.py", child("deep", 3.0))
text = child("child_a", 10.0).replace(
"block = scad.make_box_rsolid(width, 4.0, 4.0)",
'deep = scad.use("deep.py")\n block = scad.make_box_rsolid(width, 4.0, 4.0)',
)
self.write("child_a.py", text)
run_notebook(self.top)
self.write("deep.py", child("deep", 4.0))
self.assertEqual(self.edited(), ["ran", "ran", "cached", "ran"])
def test_a_path_computed_at_run_time_is_still_followed(self) -> None:
self.write("top.py", parent(first='scad.use(_name + ".py")', before='_name = "child_a"'))
run_notebook(self.top)
self.write("child_a.py", child("child_a", 12.0))
statuses = self.edited()
self.assertEqual(statuses[1], "ran")
self.assertEqual(statuses[3], "ran")
def test_the_kernel_keys_a_static_use_cell_and_runs_a_dynamic_one_live(self) -> None:
for source, expected in ((parent(), True), (parent(first='scad.use(_name + ".py")', before='_name = "child_a"'), False)):
path = self.write("probe.py", source)
config = read_notebook_config(path)
internal = InternalApp(load_notebook(path, cell_prefix=_cell_prefix(config)))
ordered = [internal.graph.cells[i] for i in internal.cell_manager.cell_ids() if i in internal.graph.cells]
setup = internal.graph.cells.get(internal.cell_manager.setup_cell_id)
digests = CellDigests(config, ordered, setup, kernel=True)
first_use = next(c for c in ordered if "use(" in c.code)
self.assertEqual(digests.digest(first_use.cell_id) is not None, expected)
+158
View File
@@ -0,0 +1,158 @@
"""Running the notebooks a cell uses side by side.
OpenCASCADE does not let go of the interpreter lock, so threads would not
help: the ``use`` calls of a cell that is about to run are started in worker
processes, which only fill the notebooks' cell caches. The cell then runs its
``use`` calls as it always did, and finds the work done. Whatever the workers
do (or fail to do), the result is the one a serial run gives.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
import unittest
from pathlib import Path
from unittest import mock
from simplecadapi.runtime import prefetch, run_notebook
from test_notebook_runtime import NotebookTestCase, notebook_text
from test_use_digests import ASSEMBLE, child, parent
class TestPlan(unittest.TestCase):
root = Path("/proj")
def plan(self, code: str, scope: dict | None = None):
return prefetch.plan_uses(code, scope or {}, self.root)
def test_literal_paths_and_values_known_before_the_cell_runs(self) -> None:
requests = self.plan(
'a = scad.use("child_a.py")\nb = scad.use("sub/b.py", id="x", WIDTH=float(W), N=3)',
{"W": 4},
)
self.assertEqual([r.path for r in requests], [self.root / "child_a.py", self.root / "sub/b.py"])
self.assertEqual((requests[1].id, requests[1].overrides), ("x", {"WIDTH": 4.0, "N": 3}))
def test_the_id_of_the_running_notebook_is_known(self) -> None:
requests = prefetch.plan_uses(
'a = scad.use("c.py", id=scad.notebook_id() + "-body", LENGTH=float(L))', {"L": 2}, self.root, notebook_id="arm"
)
self.assertEqual([(r.id, dict(r.overrides)) for r in requests], [("arm-body", {"LENGTH": 2.0})])
self.assertEqual(self.plan('a = scad.use("c.py", id=scad.notebook_id())'), []) # not told which
def test_what_cannot_be_known_is_left_to_the_cell(self) -> None:
for code in (
'a = scad.use(name + ".py")', # computed path
'a = scad.use("c.py") if flag else None', # may not run
'for n in names:\n scad.use("c.py")', # a loop
'a = scad.use("c.py", LENGTH=other.size())', # a call the plan will not make
'a = scad.use("c.py", LENGTH=missing)', # a name the cell defines itself
"def f():\n return scad.use('c.py')", # not run by the cell
):
with self.subTest(code=code):
self.assertEqual(self.plan(code, {"flag": True, "names": [1], "other": object()}), [])
def test_two_requests_for_one_run_are_one(self) -> None:
requests = self.plan('a = scad.use("c.py", W=1.0)\nb = scad.use("c.py", W=1.0)\nc = scad.use("c.py", W=2.0)')
self.assertEqual(len({r.key for r in requests}), 2)
class TestPrefetchedRun(NotebookTestCase):
def setUp(self) -> None:
super().setUp()
patcher = mock.patch.dict(os.environ, {"SCA_JOBS": "2"})
patcher.start()
self.addCleanup(patcher.stop)
self.addCleanup(prefetch.shutdown)
prefetch.reset_stats()
self.write("child_a.py", child("child_a", 10.0))
self.write("child_b.py", child("child_b", 6.0))
self.top = self.write("top.py", self.parent_text())
@staticmethod
def parent_text() -> str:
# both uses in one cell, as in a real assembly
return notebook_text(
"@app.cell\ndef _():\n a = scad.use('child_a.py')\n b = scad.use('child_b.py')\n return a, b\n",
"@app.cell\ndef _(a, b):\n" + ASSEMBLE.rstrip("\n") + "\n return (top,)\n",
table='id = "top"',
)
def test_a_cell_with_two_uses_starts_them_together_and_gives_the_serial_result(self) -> None:
run = run_notebook(self.top)
self.assertEqual(sorted(Path(p).name for p in prefetch.stats()["submitted"]), ["child_a.py", "child_b.py"])
serial = run_notebook(self.top, cache=False)
self.assertEqual(run.definition.content_hash, serial.definition.content_hash)
def test_a_notebook_that_has_not_changed_since_it_last_ran_is_not_started_again(self) -> None:
run_notebook(self.top)
prefetch.reset_stats()
run_notebook(self.top, cache=False) # the cell runs again; the children are as they were
self.assertEqual(prefetch.stats()["submitted"], [])
def stamped(self, seconds: float) -> None:
"""Make both children look like they last took *seconds* to build."""
from simplecadapi.runtime.config import read_notebook_config
for name in ("child_a.py", "child_b.py"):
stamp = prefetch._stamp_file(read_notebook_config(self.root / name))
stamp.parent.mkdir(parents=True, exist_ok=True)
stamp.write_text(json.dumps({"signature": "stale", "seconds": seconds}))
def test_notebooks_that_build_in_moments_are_not_worth_a_worker(self) -> None:
# a worker takes seconds to start; two builds of a fifth of a second do not repay it
self.stamped(0.2)
run_notebook(self.top)
self.assertEqual(prefetch.stats()["submitted"], [])
def test_notebooks_that_took_long_are_started_together(self) -> None:
self.stamped(20.0)
run_notebook(self.top)
self.assertEqual(len(prefetch.stats()["submitted"]), 2)
def test_a_run_remembers_how_long_it_took_to_build(self) -> None:
from simplecadapi.runtime.config import read_notebook_config
run_notebook(self.root / "child_a.py")
stamp = json.loads(prefetch._stamp_file(read_notebook_config(self.root / "child_a.py")).read_text())
self.assertGreater(stamp["seconds"], 0.0)
# a run that only restores does not make it look cheap
run_notebook(self.root / "child_a.py", cache=True)
again = json.loads(prefetch._stamp_file(read_notebook_config(self.root / "child_a.py")).read_text())
self.assertEqual(again["seconds"], stamp["seconds"])
def test_a_failing_child_is_reported_by_the_run_that_uses_it(self) -> None:
self.write("child_b.py", child("child_b", 6.0).replace("make_box_rsolid(width, 4.0, 4.0)", "make_box_rsolid(width, -4.0, 4.0)"))
with self.assertRaises(Exception) as caught:
run_notebook(self.top)
# the error of the child itself, not something about a worker
self.assertIn("greater than zero", str(caught.exception))
def test_one_use_is_not_worth_a_worker(self) -> None:
one = notebook_text(
"@app.cell\ndef _():\n a = scad.use('child_a.py')\n return (a,)\n",
"@app.cell\ndef _(a):\n"
' _asm = scad.make_assembly_rassembly(assembly_id="single", name="single")\n'
' single = scad.add_component_rassembly(assembly=_asm, item=a, component_id="a", '
"placement=scad.identity_placement_rplacement())\n"
" return (single,)\n",
table='id = "single"',
)
run_notebook(self.write("single.py", one))
self.assertEqual(prefetch.stats()["submitted"], [])
def test_the_command_line_prints_only_its_report(self) -> None:
chatty = child("child_a", 10.0).replace("width = 10.0", "print('from the child')\n width = 10.0")
self.write("child_a.py", chatty)
done = subprocess.run(
[sys.executable, "-c", "import sys; from simplecadapi.cli import main; sys.exit(main(['run', sys.argv[1]]))", str(self.top)],
capture_output=True, text=True, env={**os.environ, "SCA_JOBS": "2"}, timeout=300,
)
self.assertEqual(done.returncode, 0, done.stderr[-2000:])
lines = [line for line in done.stdout.splitlines() if line.strip()]
self.assertEqual(json.loads(lines[-1])["definition_id"], "top")
self.assertNotIn("from the child", done.stdout) # a worker's output goes nowhere