fix(cache): a cell's node record is kept per cache entry, so another run of the cell cannot replace it
One cell has a cache entry for every combination of values it reads (a child notebook run with other overrides, a parameter changed in the editor), all under one digest directory, and the nodes of two entries carry the same ids with other contents. The nodes were written to lineage.<cell>, so the entry run last replaced the record of the others. Restoring another entry then registered nodes that did not belong to its values: a later cell failed with 'not owned by active graph' (ids the other run lacked) or bound to the other run's node of the same id. Records are now named by the entry's key; records written by earlier versions are not read and their entries run again. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -501,7 +501,7 @@ class CellCache(CachedLifecycle):
|
||||
|
||||
try:
|
||||
# The entries of a run live in <state>/<id>/cache: the nodes of its cells belong to the graph of that id.
|
||||
return lineage.load(self._root / digest, str(cell.cell_id), self._root.parent.name)
|
||||
return lineage.load(self._root / digest, str(cell.cell_id), self._root.parent.name, self._entry_key(cell))
|
||||
except Exception as error: # noqa: BLE001 - an unreadable record is a miss, never a failure
|
||||
say(f"unreadable lineage record of cell {cell.cell_id} ({type(error).__name__}: {error}); running it again")
|
||||
return False
|
||||
@@ -525,16 +525,17 @@ class CellCache(CachedLifecycle):
|
||||
glbls[name] = None
|
||||
glbls[f"_cell_{cell.cell_id}_{name[1:]}"] = None
|
||||
live = glbls
|
||||
entry_key = self._entry_key(cell) # (marimo forgets the attempt when it tears the cell down)
|
||||
super().teardown(cell, glbls, run_result)
|
||||
if run_result.exception is None:
|
||||
self._save_lineage(cell, loader)
|
||||
self._save_lineage(cell, loader, entry_key)
|
||||
if attempt is not None and not attempt.hit and self._memory is not None:
|
||||
# what this run made is what the entry names: the next evaluation with the same inputs takes it
|
||||
values = {var: (None if var.startswith("_") else live.get(lookup)) for var, lookup in attempt.contextual_defs()}
|
||||
if not any(is_unhashable_stub(value) for value in values.values()):
|
||||
self._memory.remember(loader, attempt.key, values, dict(attempt.meta), attempt.stateful_refs)
|
||||
|
||||
def _save_lineage(self, cell: CellImpl, loader: LazyLoader) -> None:
|
||||
def _save_lineage(self, cell: CellImpl, loader: LazyLoader, entry_key: str | None) -> None:
|
||||
from . import executor, lineage
|
||||
|
||||
session = executor.take_session(cell.cell_id)
|
||||
@@ -543,7 +544,13 @@ class CellCache(CachedLifecycle):
|
||||
return # a restored cell keeps the record it was restored with
|
||||
if session.graph.graph_id != self._root.parent.name:
|
||||
return # recorded under another id: not this run's lineage to keep
|
||||
lineage.save(self._root / digest, str(cell.cell_id), session)
|
||||
lineage.save(self._root / digest, str(cell.cell_id), session, entry_key)
|
||||
|
||||
def _entry_key(self, cell: CellImpl) -> str | None:
|
||||
"""The key of the entry this cell restores or writes: its record of nodes is named by it (see :mod:`.lineage`)."""
|
||||
|
||||
attempt = self._attempts.get(cell.cell_id)
|
||||
return getattr(attempt, "hash", None)
|
||||
|
||||
def _invalidate(self, cell_id: CellId_t) -> None:
|
||||
# The producer's entry lives in the store of its own digest.
|
||||
|
||||
@@ -6,6 +6,11 @@ a cell are written beside the cell's entry (``lineage.<cell key>``) and put
|
||||
back into the process-wide node registry when the cell is restored. Cells
|
||||
restore in dependency order, so the nodes a cell's own nodes are built on are
|
||||
registered by then.
|
||||
|
||||
A record belongs to one cache *entry* of the cell, not to the cell: one cell has an entry for every combination of the
|
||||
values it reads (a child notebook run with other overrides, a parameter changed in the editor), all under one digest
|
||||
directory, and the nodes of two entries carry the same ids (the ids count from the cell's code) with other contents.
|
||||
The record is named by the entry's key, so restoring an entry registers the nodes that entry's values name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -23,8 +28,12 @@ def _own(node_id: str, namespace: str) -> bool:
|
||||
return node_id.startswith(namespace + "_") or node_id.startswith(_EXTERNAL)
|
||||
|
||||
|
||||
def save(directory: Path, cell_id: str, session: Any) -> None:
|
||||
"""Write the nodes *session* recorded (not the ones it adopted) beside the cell's entry."""
|
||||
def _record(directory: Path, cell_id: str, key: str | None) -> Path:
|
||||
return directory / (f"{_PREFIX}{cell_id}" if key is None else f"{_PREFIX}{cell_id}.{key}")
|
||||
|
||||
|
||||
def save(directory: Path, cell_id: str, session: Any, key: str | None = None) -> None:
|
||||
"""Write the nodes *session* recorded (not the ones it adopted) beside the cell's entry *key*."""
|
||||
|
||||
namespace = session.graph.namespace
|
||||
payload = session.graph.to_dict()
|
||||
@@ -38,14 +47,14 @@ def save(directory: Path, cell_id: str, session: Any) -> None:
|
||||
"frames": session.frame_graph.to_dict(),
|
||||
}
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
target = directory / f"{_PREFIX}{cell_id}"
|
||||
target = _record(directory, cell_id, key)
|
||||
partial = target.with_suffix(".part")
|
||||
partial.write_bytes(zlib.compress(pickle.dumps(document, protocol=5), 1))
|
||||
partial.replace(target)
|
||||
|
||||
|
||||
def load(directory: Path, cell_id: str, graph_id: str | None = None) -> bool:
|
||||
"""Register the nodes saved for the cell; ``False`` when there is no entry.
|
||||
def load(directory: Path, cell_id: str, graph_id: str | None = None, key: str | None = None) -> bool:
|
||||
"""Register the nodes saved for the cell's entry *key*; ``False`` when there is no record.
|
||||
|
||||
With *graph_id*, a record of another graph is no entry either: its nodes would be registered under that graph,
|
||||
and the values restored beside it (which name their nodes by this run's graph) would find none.
|
||||
@@ -53,7 +62,7 @@ def load(directory: Path, cell_id: str, graph_id: str | None = None) -> bool:
|
||||
|
||||
import simplecadapi._cpp as native
|
||||
|
||||
target = directory / f"{_PREFIX}{cell_id}"
|
||||
target = _record(directory, cell_id, key)
|
||||
try:
|
||||
document = pickle.loads(zlib.decompress(target.read_bytes()))
|
||||
except OSError:
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""A cell's nodes are restored by the cache entry the values came from, not by the cell.
|
||||
|
||||
A cell has one cache entry per combination of the values it reads (an assembly that ``use`` s a part with other
|
||||
overrides runs the part's cells under the same graph id, digest and cell, with other inputs). The ids of the nodes
|
||||
count from the cell's code, so two entries carry the same ids with other contents. The nodes were written beside the
|
||||
cell, so the entry run last replaced the record of the others: restoring another entry then registered nodes that
|
||||
did not belong to its values, and a later cell that read them failed with "not owned by active graph" (the ids the
|
||||
other run did not have) or bound to the other run's node of the same id (the ids it had).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("marimo")
|
||||
|
||||
import simplecadapi as scad
|
||||
import simplecadapi._cpp as native
|
||||
from simplecadapi.runtime import runner
|
||||
from test_notebook_runtime import cell, notebook_text
|
||||
|
||||
PART = notebook_text(
|
||||
cell("LINK_L = 60.0", returns="LINK_L"),
|
||||
cell("STEPS = 4", returns="STEPS"),
|
||||
cell(
|
||||
"""
|
||||
beam = scad.make_box_rsolid(LINK_L, 10.0, 5.0)
|
||||
for _i in range(STEPS):
|
||||
beam = scad.union_rsolid(beam, scad.make_cylinder_rsolid(2.0, 8.0))
|
||||
""",
|
||||
reads="LINK_L, STEPS",
|
||||
returns="beam",
|
||||
),
|
||||
cell('link = scad.Part(part_id="link", body=beam)', reads="beam", returns="link"),
|
||||
table='id = "link"',
|
||||
)
|
||||
|
||||
|
||||
def cut_like_a_cell(beam):
|
||||
"""What the next cell of the notebook does with the value (a cell session that accepts other cells' nodes)."""
|
||||
|
||||
with scad.GraphSession(graph_id="link", namespace="cCUT", shared_lineage=True) as session:
|
||||
scad.cut_rsolid(beam, scad.make_cylinder_rsolid(1.0, 20.0))
|
||||
return session.graph
|
||||
|
||||
|
||||
class EntryRecordTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.path = Path(tmp.name).resolve() / "link.py"
|
||||
self.path.write_text(PART, encoding="utf-8")
|
||||
native.forget_graph_nodes("link")
|
||||
self.addCleanup(native.forget_graph_nodes, "link")
|
||||
|
||||
def after_another_run(self, **overrides):
|
||||
"""The editor's own values, restored from the disk after a child run with *overrides* wrote its entries."""
|
||||
|
||||
runner.run_notebook(self.path)
|
||||
runner.run_notebook(self.path, overrides=overrides)
|
||||
native.forget_graph_nodes("link") # a new process: only the records of the disk register nodes
|
||||
run = runner.run_notebook(self.path)
|
||||
self.assertEqual({report.status for report in run.cells[1:]}, {"cached"})
|
||||
return run.values["beam"]
|
||||
|
||||
def test_values_restored_after_a_run_with_fewer_nodes_find_their_nodes(self) -> None:
|
||||
graph = cut_like_a_cell(self.after_another_run(STEPS=1))
|
||||
self.assertIn("make_union_rsolid", {node.op for node in graph.nodes})
|
||||
self.assertEqual(sum(node.op == "make_union_rsolid" for node in graph.nodes), 4)
|
||||
|
||||
def test_values_restored_after_a_run_with_the_same_nodes_bind_to_their_own(self) -> None:
|
||||
graph = cut_like_a_cell(self.after_another_run(LINK_L=90.0))
|
||||
boxes = [node for node in graph.nodes if node.op == "make_box_rsolid"]
|
||||
self.assertEqual([box.params["width"] for box in boxes], [60.0])
|
||||
Reference in New Issue
Block a user