Files
SkillFactory/scripts/standing_acceptance.py
T
chenlin 093ab883be feat: release guided Go2 standing skill with explicit drift waiver
Add loaded-standing physics, PPO training, fixed-candidate acceptance, isolated inference, and fail-closed publication. Preserve the original 120-second drift failure and bind the operator-approved exception to the exact candidate and evidence.

Include pending Reach training/release infrastructure and archived packages. Leave pre-existing short-worker visualization changes and large generated training artifacts out of this commit.
2026-10-12 17:26:03 +08:00

355 lines
16 KiB
Python

"""Fail-closed standing evidence audit. Re-read traces and exercise CPU runtime; no simulation."""
import argparse
import json
import math
import re
import traceback
from pathlib import Path
from runtime_common import critical_log_lines, write_json
def read_json(path):
def invalid(value):
raise ValueError(f"Nonfinite JSON: {value}")
return json.loads(Path(path).read_text(), parse_constant=invalid)
def require(condition, message):
if not condition:
raise ValueError(message)
def check_process(result, process, log, expected_success):
require(
process.get("cleanup", {}).get("clean") is True and process["cleanup"]["remaining"] == [], "Unclean process"
)
critical = critical_log_lines(log)
require(critical == process["critical_log_lines"], "Critical log report mismatch")
text = Path(log).read_text(errors="replace")
require(not re.search(r"Traceback|buffer\s+overflow|CUDA.*out.of.memory", text, re.I), "Failed runtime log")
require("error" not in result and "shutdown_error" not in result, "Trial exception/shutdown failure")
if expected_success:
require(result["passed"] is True and result["status"] == "completed", "Failed required trial")
require(
process["passed"] is True
and process["status"] == "completed"
and process["returncode"] == 0
and not critical,
"Failed required process",
)
elif result["passed"]:
require(
process["passed"] and process["returncode"] == 0 and not critical, "Baseline process failed unexpectedly"
)
else:
require(
result["status"] == "failed" and process["passed"] is False and process["returncode"] in (1, 2),
"Invalid controlled failure",
)
require(
all(
line.startswith("[ERROR] Command failed with code 1:") and "scripts/evaluate_standing.py" in line
for line in critical
),
"Unexpected critical failure",
)
def memory_gate(samples):
require([x["sim_seconds"] for x in samples] == list(range(1, 121)), "Missing stability memory samples")
require(all(type(x["bytes"]) is int and x["bytes"] > 0 for x in samples), "Invalid memory measurement")
first = max(x["bytes"] for x in samples if 2 <= x["sim_seconds"] <= 12)
last = max(x["bytes"] for x in samples if 110 <= x["sim_seconds"] <= 120)
return last / first - 1
def verify_trials(run, package, waiver_path=None, waiver_sha256=None):
import torch
from SkillFactory.legs.frozen_standing import FrozenStanding, checked_manifest, file_hash
from standing_deviation import apply_drift_waiver, load_waiver
from standing_protocol import inputs_for, reduce_trace
torch.set_num_threads(4)
lock = read_json(run / "candidate_lock.json")
manifest_hash = file_hash(package / "manifest.json")
manifest = checked_manifest(package, lock["manifest_sha256"])
require(
lock["passed"] and lock["actor_extraction_max_error"] <= 1e-6 and lock["selected_update"] == 1024,
"Invalid candidate extraction",
)
require(manifest["files"] == lock["core_files"], "Candidate files changed")
b = read_json(package / "binding.json")
cfg = read_json(package / "task.json")
policy = FrozenStanding(
package, manifest_sha256=manifest_hash, task_sha256=b["task_sha256"], support_sha256=b["support_sha256"]
)
independent = read_json(run / "independent_candidate.json")
require(
independent["passed"]
and independent["status"] == "completed"
and independent["isolated"]
and independent["package_only"]
and independent["manifest_sha256"] == manifest_hash
and independent["actor_sha256"] == file_hash(package / "actor.pt")
and independent["deterministic_batches"] == [1, 16, 64, 256]
and independent["calls_per_batch"] == 100
and independent["invalid_inputs_binding_corruption_rejected"]
and independent["input_state_unchanged"]
and not independent["forbidden_imports"]
and independent["requires_grad"] is False,
"Independent candidate load failed",
)
waiver = load_waiver(waiver_path, waiver_sha256, run, package)
trials, artifacts, waiver_receipts = {}, {}, []
source_bundle = read_json(Path(cfg["source_bundle"]) / "manifest.json")
expected_mass = read_json(Path("logs/loaded_stand/v0/standing_clean/result.json"))["total_mass_kg"]
for name, mode, controller in (
("full", "full", "policy"),
("holdout", "holdout", "policy"),
("hold", "holdout", "hold"),
("stability", "stability", "policy"),
("exploratory", "exploratory", "policy"),
):
root = run / name
report = read_json(root / "result.json")
process = read_json(run / f"{name}_process.json")
meta = read_json(root / "metadata.json")
require(
report["mode"] == mode and report["controller"] == controller and report["metadata"] == meta,
"Trial identity/metadata mismatch",
)
require(meta["inputs"] == inputs_for(mode), "Wrong holdout/full/stability protocol")
require(
report["package_manifest_sha256"] == manifest_hash
and report["actor_sha256"] == file_hash(package / "actor.pt"),
"Physics tested different policy identity",
)
require(report["physics_trace_sha256"] == file_hash(root / "physics_trace.pt"), "Trace changed")
require(
abs(meta["total_mass_kg"] - expected_mass) < 1e-6 and abs(meta["height_m"] - b["height_m"]) < 1e-8,
"Rig changed",
)
require(
meta["limits"] == [[j["limits"]["lower"], j["limits"]["upper"]] for j in source_bundle["joints"]]
and meta["speeds"] == [j["limits"]["velocity"] for j in source_bundle["joints"]]
and meta["efforts"]
== torch.tensor([j["limits"]["effort"] for j in source_bundle["joints"]], dtype=torch.float32).tolist(),
"Hardware limits changed",
)
expected_sources = set(b["training_identity"]["sources"]) | {
"scripts/evaluate_standing.py",
"scripts/standing_protocol.py",
"scripts/runtime_common.py",
"scripts/supervise_sim.py",
}
require(set(report["source_hashes"]) == expected_sources, "Incomplete execution source identity")
for path, digest in report["source_hashes"].items():
require(
file_hash(path) == digest == file_hash(root / "source_snapshot" / path), "Evaluation source changed"
)
data = torch.load(root / "physics_trace.pt", map_location="cpu", weights_only=True)
reduced = reduce_trace(data, meta, cfg)
del data
require(
report["physical_samples"] == reduced["physical_samples"] and report["rows"] == reduced["rows"],
"Reduction mismatch",
)
require(report["mean_height_error_m"] == reduced["mean_height_error_m"], "Metric mismatch")
gates = reduced["gates"].copy()
if mode == "full":
for key in ("indexed_reset", "timeout_terminal_reset", "bad_action_atomic_rejection"):
require(report["gates"].get(key) is True, f"Missing real reset/action probe: {key}")
gates[key] = True
if mode == "stability":
growth = memory_gate(read_json(root / "memory.json"))
require(growth == report["memory_growth_fraction"], "Memory report mismatch")
gates["memory"] = growth <= 0.05
require(gates == report["gates"] and all(gates.values()) == report["passed"], "Gates/declared pass mismatch")
required = name in ("full", "holdout", "stability")
waived = name == "stability" and waiver is not None
if waived:
waiver_receipts.append(apply_drift_waiver(waiver, gates, reduced["rows"], cfg, meta))
# A waiver covers a metric, never backend errors, incomplete traces or bad cleanup.
check_process(report, process, run / f"{name}.log", required and not waived)
if required:
require(all(gates.values()) or waived, f"Required physics failed: {name}")
samples = torch.load(root / "inference_samples.pt", map_location="cpu", weights_only=True)
if controller == "policy":
with torch.inference_mode():
error = (policy.act(samples["observations"]) - samples["actions"]).abs().max().item()
require(math.isfinite(error) and error <= 1e-5, f"CPU/CUDA inference mismatch: {error}")
else:
require((samples["actions"] == 0).all().item(), "Hold baseline is not zero-action")
error = 0.0
trials[name] = {
"passed": report["passed"],
"accepted_by_operator_waiver": waived,
"gates": gates,
"mean_height_error_m": reduced["mean_height_error_m"],
"cpu_cuda_max_error": error,
"physical_samples": reduced["physical_samples"],
"max_height_error_m": max(r["max_height_error_m"] for r in report["rows"]),
"max_tilt_rad": max(r["max_tilt_rad"] for r in report["rows"]),
"max_foot_drift_m": max(r["max_foot_drift_m"] for r in report["rows"]),
"min_contact_fraction": min(r["contact_fraction"] for r in report["rows"]),
"sensor_force_error_N": reduced["sensor_static_max_force_error_N"],
"sensor_torque_error_Nm": reduced["sensor_static_max_torque_error_Nm"],
"memory_growth_fraction": report.get("memory_growth_fraction"),
}
for p in (
root / "result.json",
root / "metadata.json",
root / "physics_trace.pt",
root / "inference_samples.pt",
root / "memory.json",
run / f"{name}_process.json",
run / f"{name}.log",
):
artifacts[str(p.resolve())] = file_hash(p)
if waiver is not None:
require(len(waiver_receipts) == 1, "Waiver must apply exactly once")
artifacts[str(Path(waiver_path).resolve())] = waiver_sha256
improvement = 1 - trials["holdout"]["mean_height_error_m"] / trials["hold"]["mean_height_error_m"]
require(improvement >= 0.30, "Holdout improvement <30%")
return {
"trials": trials,
"holdout_improvement_fraction": improvement,
"artifacts": artifacts,
"candidate_manifest_sha256": manifest_hash,
"core_files": lock["core_files"],
"acceptance_basis": "operator_waiver" if waiver is not None else "all_required_physics_gates",
"unwaived_required_physics_passed": all(trials[k]["passed"] for k in ("full", "holdout", "stability")),
"waiver": waiver,
"waiver_sha256": waiver_sha256,
"waiver_receipts": waiver_receipts,
}
def verify_training(package):
from package_standing import SELECTED_SHA256
from SkillFactory.legs.frozen_standing import file_hash
from SkillFactory.legs.standing_ppo import canonical_hash
run = Path("logs/loaded_stand/train_v1")
result = read_json(run / "result.json")
config = read_json(run / "config.json")
binding = read_json(package / "binding.json")
require(
result["formal_training_complete"]
and result["passed"]
and result["status"] == "completed"
and result["update"] == config["updates"] == 2048
and result["transitions"] == 16777216,
"Incomplete training",
)
require(
result["identity"] == binding["training_identity"] and result["identity"]["config"] == canonical_hash(config),
"Training identity mismatch",
)
require(
binding["checkpoint_sha256"] == SELECTED_SHA256 == file_hash(run / "checkpoints/best_validation.pt"),
"Selected checkpoint changed",
)
rows = [json.loads(line) for line in (run / "metrics.jsonl").read_text().splitlines()]
train = [r for r in rows if r["kind"] == "train"]
require(
[r["update"] for r in train] == list(range(1, 2049))
and all(r["transitions"] == r["update"] * 8192 for r in train),
"Training sequence incomplete",
)
values = [read_json(p) for p in sorted((run / "validation").glob("update_*.json"))]
require([r["update"] for r in values] == list(range(256, 2049, 256)), "Missing validation")
best = min(values, key=lambda r: (r["score"], r["update"]))
require(
best["update"] == 1024 and read_json(package / "selection.json")["score"] == best["score"],
"Not validation-selected",
)
require(
all(r["input_hash"] == canonical_hash(r["inputs"]) == values[0]["input_hash"] for r in values),
"Validation inputs changed",
)
for path, digest in result["identity"]["sources"].items():
require(file_hash(path) == digest, "Training-bound source changed")
check_process(result, read_json(run.parent / "train_v1_process.json"), run.parent / "train_v1.log", True)
return {"updates": 2048, "transitions": 16777216, "selected_update": 1024, "checkpoint_sha256": SELECTED_SHA256}
def audit(run, output, waiver_path=None, waiver_sha256=None, engineering_path=None):
from SkillFactory.legs.frozen_standing import file_hash
write_json(output, {"passed": False, "status": "running", "accepted_skill": False})
try:
package = run / "candidate"
training = verify_training(package)
evidence = verify_trials(run, package, waiver_path, waiver_sha256)
engineering_path = Path(engineering_path) if engineering_path else run / "engineering.json"
engineering = read_json(engineering_path)
require(engineering["passed"] is True and engineering["review_passed"] is True, "Engineering/review not passed")
require(
re.search(r"(?i)(?:merge verdict|verdict).{0,8}\bOK\b", Path(engineering["review_report"]).read_text()),
"Missing positive final review verdict",
)
require(
{
"scripts/standing_acceptance.py",
"scripts/publish_standing.py",
"scripts/verify_frozen_standing.py",
"tests/test_standing_acceptance.py",
engineering["review_report"],
engineering["pytest_log"],
engineering["ruff_log"],
}.issubset(engineering["files"]),
"Incomplete engineering identity",
)
if waiver_path is not None:
require("scripts/standing_deviation.py" in engineering["files"], "Missing waiver code review identity")
for path, digest in engineering["files"].items():
require(file_hash(path) == digest, "Engineering evidence changed")
require("All checks passed!" in Path(engineering["ruff_log"]).read_text(), "Ruff not passed")
require(
re.search(r"\d+ passed", Path(engineering["pytest_log"]).read_text())
and not re.search(r"\d+ failed|ERROR collecting", Path(engineering["pytest_log"]).read_text()),
"Tests failed",
)
root = run.parent
wall = sum(read_json(p)["wall_seconds"] for p in root.rglob("*_process.json"))
size = sum(p.stat().st_size for p in root.rglob("*") if p.is_file())
require(wall <= 1800 and size < 2 * 1024**3, "Acceptance resource budget exceeded")
evidence.update(
passed=True,
status="completed",
accepted_skill=False,
stage="eligible_for_frozen_publication",
training=training,
physics_wall_seconds=wall,
artifact_bytes=size,
)
evidence["engineering_path"] = str(engineering_path.resolve())
for path in (engineering_path, run / "candidate_lock.json", run / "independent_candidate.json"):
evidence["artifacts"][str(path.resolve())] = file_hash(path)
write_json(output, evidence)
return evidence
except BaseException:
write_json(
output, {"passed": False, "status": "failed", "accepted_skill": False, "error": traceback.format_exc()}
)
raise
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--waiver", type=Path)
parser.add_argument("--waiver-sha256")
parser.add_argument("--engineering", type=Path)
args = parser.parse_args()
audit(args.run, args.output, args.waiver, args.waiver_sha256, args.engineering)
if __name__ == "__main__":
main()