Files
SkillFactory/tests/test_standing_acceptance.py
T
chenlin 093ab883be feat: release guided Go2 standing skill with explicit drift waiver
Add loaded-standing physics, PPO training, fixed-candidate acceptance, isolated inference, and fail-closed publication. Preserve the original 120-second drift failure and bind the operator-approved exception to the exact candidate and evidence.

Include pending Reach training/release infrastructure and archived packages. Leave pre-existing short-worker visualization changes and large generated training artifacts out of this commit.
2026-10-12 17:26:03 +08:00

277 lines
11 KiB
Python

"""Pure regressions: frozen standing ABI and independent protocol reducer."""
import json
import sys
from pathlib import Path
import pytest
import torch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
from SkillFactory.legs.frozen_standing import CORE, SHAPES, Actor, FrozenStanding, file_hash
from SkillFactory.legs.standing_ppo import canonical_hash
from SkillFactory.legs.support import disturbance_wrench
from standing_protocol import WIDTHS, calibration_errors, inputs_for, load_at, reduce_trace
@pytest.fixture
def package(tmp_path):
task = tmp_path / "task.json"
task.write_text("{}")
binding = {
"task_sha256": file_hash(task),
"support_sha256": "a" * 64,
"height_m": 0.33,
"nominal": [0.0, 0.8, -1.5],
"action_scale_rad": 0.5,
"soft_limits": [[-0.8, 0.8], [-1.0, 3.0], [-2.6, -0.9]],
}
for name in CORE - {"task.json", "actor.pt", "binding.json"}:
(tmp_path / name).write_text("fixture")
(tmp_path / "binding.json").write_text(json.dumps(binding))
torch.save(Actor().state_dict(), tmp_path / "actor.pt")
manifest = {
"format": "skillfactory.standing-bundle.v1",
"contract": "go2-fl-loaded-stand-v0",
"version": "1.0.0",
"hidden": 128,
"files": {p.name: file_hash(p) for p in tmp_path.iterdir()},
}
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
return tmp_path, {
"manifest_sha256": file_hash(tmp_path / "manifest.json"),
"task_sha256": binding["task_sha256"],
"support_sha256": binding["support_sha256"],
}
def observation(batch=3):
obs = {k: torch.zeros(batch, *shape) for k, shape in SHAPES.items()}
obs["goal"][:] = 0.33
obs["base_state"][:, 6] = 1
return obs
def test_frozen_abi_stateless_and_rejects_bad_contract(package):
path, kwargs = package
p = FrozenStanding(path, **kwargs)
obs = observation()
before = {k: v.clone() for k, v in obs.items()}
a = p.act(obs)
assert a.shape == (3, 3) and not a.requires_grad and torch.equal(a, p.act(obs))
assert all(torch.equal(v, before[k]) for k, v in obs.items())
for key in kwargs:
with pytest.raises(ValueError):
FrozenStanding(path, **{**kwargs, key: "0" * 64})
for key in obs:
bad = {k: v.clone() for k, v in obs.items()}
bad[key].flatten()[0] = torch.nan
with pytest.raises(ValueError):
p.act(bad)
for key, value in (
("goal", torch.zeros(3, 2)),
("goal", torch.zeros(3, 1, dtype=torch.float64)),
("goal", torch.zeros(3, 1, device="meta")),
):
with pytest.raises(ValueError):
p.act({**obs, key: value})
with pytest.raises(ValueError):
p.act(observation(0))
(path / "actor.pt").write_bytes(b"corrupted")
with pytest.raises(ValueError, match="integrity"):
FrozenStanding(path, **kwargs)
def test_required_manifest_entries_cannot_be_deleted(package):
path, kwargs = package
m = json.loads((path / "manifest.json").read_text())
del m["files"]["actor.pt"]
(path / "manifest.json").write_text(json.dumps(m))
with pytest.raises(ValueError, match="Missing"):
FrozenStanding(path, **{**kwargs, "manifest_sha256": file_hash(path / "manifest.json")})
def test_protocol_fixed_coverage_and_original_load_equivalence():
full = inputs_for("full")
for time in (0.0, 3.2, 4.0, 5.75, 10.0, 13.99):
t = torch.full((13,), time)
actual = load_at(
t,
torch.tensor(full["basis"]),
torch.ones(13),
torch.zeros(13),
torch.tensor([5.0, 5.0, 10.0, 1.0, 1.0, 1.0]),
)
expected = disturbance_wrench(t, torch.arange(13), torch.tensor([5.0, 5.0, 10.0, 1.0, 1.0, 1.0]))
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
h = inputs_for("holdout")
assert h == inputs_for("holdout") and h["seed"] == 83000
assert len(set(map(tuple, h["basis"]))) == 13
e = torch.tensor(inputs_for("exploratory")["basis"])
assert (e.abs().sum(-1) == 2).all() and len(e.unique(dim=0)) == 60
def test_calibration_cannot_average_away_or_cancel_a_bad_cycle():
expected = torch.zeros(21000, 1, 6)
measured = expected.clone()
measured[999:1499, :, 2] = -10
measured[7999:8499, :, 2] = 10
error, counts = calibration_errors(measured, expected, 42.0)
assert counts == [500] * 9
assert error[0, 0, 2] == 10 and error[3, 0, 2] == 10
assert error.max() > 1.5
def test_release_guards_reject_stale_success_and_missing_memory(tmp_path):
from standing_acceptance import check_process, memory_gate, read_json
log = tmp_path / "bad.log"
log.write_text("[Error] backend exploded\n")
result = {"passed": True, "status": "completed"}
process = {
"passed": True,
"status": "completed",
"returncode": 0,
"cleanup": {"clean": True, "remaining": []},
"critical_log_lines": ["[Error] backend exploded"],
}
with pytest.raises(ValueError, match="process"):
check_process(result, process, log, True)
with pytest.raises(ValueError, match="memory"):
memory_gate([])
values = [{"sim_seconds": i, "bytes": 1000 if i < 110 else 1100} for i in range(1, 121)]
assert memory_gate(values) > 0.05
bad = tmp_path / "invalid.json"
bad.write_text('{"passed": true, "value": NaN}')
with pytest.raises(ValueError, match="Nonfinite"):
read_json(bad)
def test_atomic_publication_never_overwrites_existing_directory(tmp_path):
from publish_standing import rename_no_replace
source, target = tmp_path / "stage", tmp_path / "version"
source.mkdir()
target.mkdir()
(target / "sentinel").write_text("keep")
with pytest.raises(OSError):
rename_no_replace(source, target)
assert source.exists() and (target / "sentinel").read_text() == "keep"
def test_failed_gate_leaves_no_release_or_lock(tmp_path, monkeypatch):
import publish_standing as publish
def reject(*args):
raise ValueError("No physics evidence")
monkeypatch.setattr(publish, "audit", reject)
with pytest.raises(ValueError, match="physics"):
publish.publish(tmp_path / "missing", tmp_path / "releases", Path(sys.executable))
releases = tmp_path / "releases"
assert not (releases / publish.NAME).exists()
assert not list(releases.glob(".*.publish.lock"))
assert not json.loads((releases / f"{publish.NAME}.acceptance.json").read_text())["passed"]
@pytest.mark.parametrize("fail_at", [1, 2, 3, 4])
def test_load_and_postcommit_failures_never_leave_accepted_paths(tmp_path, monkeypatch, fail_at):
import publish_standing as publish
run = tmp_path / "run"
package = run / "candidate"
package.mkdir(parents=True)
(package / "actor.pt").write_bytes(b"fixture")
files = {"actor.pt": file_hash(package / "actor.pt")}
(package / "manifest.json").write_text(json.dumps({"files": files}))
evidence = {
"passed": True,
"stage": "eligible_for_frozen_publication",
"candidate_manifest_sha256": file_hash(package / "manifest.json"),
"core_files": files,
"artifacts": {},
"trials": {},
"training": {},
"holdout_improvement_fraction": 1.0,
}
for name in ("publication_gate", "independent_candidate", "candidate_lock"):
(run / f"{name}.json").write_text("{}")
log = run / "checks.log"
log.write_text("fixture")
(run / "engineering.json").write_text(
json.dumps(dict.fromkeys(("pytest_log", "ruff_log", "review_report"), str(log)))
)
for name in ("full", "holdout", "hold", "stability", "exploratory"):
(run / name).mkdir()
(run / name / "result.json").write_text("{}")
(run / f"{name}_process.json").write_text("{}")
monkeypatch.setattr(publish, "audit", lambda *args: evidence)
calls = 0
def check(directory, *args):
nonlocal calls
calls += 1
if calls == fail_at:
raise ValueError("Injected publication failure")
return {"passed": True, "manifest_sha256": file_hash(directory / "manifest.json")}
writer = publish.write_json
def write(path, value):
if fail_at == 4 and path.name.endswith(".acceptance.json") and value.get("passed"):
raise ValueError("Injected publication failure")
writer(path, value)
monkeypatch.setattr(publish, "verify", check)
monkeypatch.setattr(publish, "write_json", write)
releases = tmp_path / "releases"
with pytest.raises(ValueError, match="Injected"):
publish.publish(run, releases, Path(sys.executable))
assert not (releases / publish.NAME).exists()
assert not (releases / f"{publish.NAME}.tar.gz").exists()
assert not list(releases.glob(".*.stage-*")) and not list(releases.glob(".*.publish.lock"))
report = json.loads((releases / f"{publish.NAME}.acceptance.json").read_text())
assert not report["passed"]
if fail_at > 1:
assert Path(report["quarantine"]).is_dir()
assert (Path(report["quarantine"]) / publish.NAME / "actor.pt").read_bytes() == b"fixture"
if fail_at == 4:
assert (Path(report["quarantine"]) / f"{publish.NAME}.tar.gz").is_file()
def test_version_race_cannot_overwrite_winner_report(tmp_path, monkeypatch):
import publish_standing as publish
releases = tmp_path / "releases"
original_open = publish.os.open
winner = b'{"passed":true,"owner":"other invocation"}\n'
def race_open(path, *args):
descriptor = original_open(path, *args)
(releases / publish.NAME).mkdir()
(releases / f"{publish.NAME}.acceptance.json").write_bytes(winner)
return descriptor
monkeypatch.setattr(publish.os, "open", race_open)
with pytest.raises(ValueError, match="Immutable"):
publish.publish(tmp_path / "unused", releases, Path(sys.executable))
assert (releases / f"{publish.NAME}.acceptance.json").read_bytes() == winner
assert (releases / publish.NAME).is_dir()
assert not list(releases.glob(".*.publish.lock"))
def test_trace_reducer_rejects_missing_and_forged_protocol():
cfg = json.loads(Path("configs/standing/go2_fl_support_v0.json").read_text())
i = inputs_for("full")
meta = {"inputs": i, "input_hash": canonical_hash(i)}
with pytest.raises(ValueError, match="trace"):
reduce_trace({}, meta, cfg)
data = {k: torch.zeros(7000, 13, width) for k, width in WIDTHS.items()}
meta["input_hash"] = "0" * 64
with pytest.raises(ValueError, match="Protocol"):
reduce_trace(data, meta, cfg)
data["base"][0, 0, 0] = torch.nan
with pytest.raises(ValueError, match="trace"):
reduce_trace(data, meta, cfg)