70 lines
2.8 KiB
Python
70 lines
2.8 KiB
Python
"""A valid checkpoint must not silently replay with changed experiment/code."""
|
|
|
|
import pytest
|
|
|
|
from universal_humanoid import artifacts
|
|
|
|
|
|
def completed_run(tmp_path):
|
|
checkpoint = tmp_path / "final.pt"
|
|
checkpoint.write_bytes(b"trusted-test-checkpoint")
|
|
experiment = {"task": "g1.flat"}
|
|
run = {
|
|
"status": "completed",
|
|
"experiment": experiment,
|
|
"experiment_sha256": artifacts.canonical_hash(experiment),
|
|
"checkpoints": {"final.pt": artifacts.digest(checkpoint)},
|
|
"provenance": {"identity_sha256": "original"},
|
|
}
|
|
artifacts.write_json(tmp_path / "run.json", run)
|
|
return checkpoint, run
|
|
|
|
|
|
def test_source_drift_is_rejected(tmp_path, monkeypatch):
|
|
checkpoint, run = completed_run(tmp_path)
|
|
monkeypatch.setattr(artifacts, "provenance", lambda _: {"identity_sha256": "original"})
|
|
assert artifacts.load_run(checkpoint) == run
|
|
monkeypatch.setattr(artifacts, "provenance", lambda _: {"identity_sha256": "changed"})
|
|
with pytest.raises(ValueError, match="changed since training"):
|
|
artifacts.load_run(checkpoint)
|
|
|
|
|
|
def test_checkpoint_replacement_is_rejected(tmp_path):
|
|
checkpoint, _ = completed_run(tmp_path)
|
|
checkpoint.write_bytes(b"different weights")
|
|
with pytest.raises(ValueError, match="Checkpoint hash"):
|
|
artifacts.load_run(checkpoint)
|
|
|
|
|
|
def test_experiment_mutation_is_rejected(tmp_path):
|
|
checkpoint, run = completed_run(tmp_path)
|
|
run["experiment"]["task"] = "t1.flat"
|
|
artifacts.write_json(tmp_path / "run.json", run)
|
|
with pytest.raises(ValueError, match="Experiment hash"):
|
|
artifacts.load_run(checkpoint)
|
|
|
|
|
|
def test_partial_training_is_not_completion(tmp_path):
|
|
checkpoint, run = completed_run(tmp_path)
|
|
run["status"] = "training"
|
|
artifacts.write_json(tmp_path / "run.json", run)
|
|
with pytest.raises(ValueError, match="Only completed"):
|
|
artifacts.load_run(checkpoint)
|
|
|
|
|
|
def test_only_sealed_intermediate_checkpoint_can_be_resumed(tmp_path, monkeypatch):
|
|
checkpoint, run = completed_run(tmp_path)
|
|
monkeypatch.setattr(artifacts, "provenance", lambda _: {"identity_sha256": "original"})
|
|
run["status"] = "interrupted"
|
|
artifacts.write_json(tmp_path / "run.json", run)
|
|
with pytest.raises(ValueError, match="Only completed"):
|
|
artifacts.load_run(checkpoint, allow_incomplete=True)
|
|
run["checkpoint_metadata"] = {"final.pt": {"committed": True, "completed_iterations": 2}}
|
|
artifacts.write_json(tmp_path / "run.json", run)
|
|
assert artifacts.load_run(checkpoint, allow_incomplete=True)["status"] == "interrupted"
|
|
with pytest.raises(ValueError, match="Only completed"):
|
|
artifacts.load_run(checkpoint) # export still requires a completed run
|
|
checkpoint.write_bytes(b"partial overwrite")
|
|
with pytest.raises(ValueError, match="Checkpoint hash"):
|
|
artifacts.load_run(checkpoint, allow_incomplete=True)
|