"""Fail-closed standing evidence audit. Re-read traces and exercise CPU runtime; no simulation.""" import argparse import json import math import re import traceback from pathlib import Path from runtime_common import critical_log_lines, write_json def read_json(path): def invalid(value): raise ValueError(f"Nonfinite JSON: {value}") return json.loads(Path(path).read_text(), parse_constant=invalid) def require(condition, message): if not condition: raise ValueError(message) def check_process(result, process, log, expected_success): require( process.get("cleanup", {}).get("clean") is True and process["cleanup"]["remaining"] == [], "Unclean process" ) critical = critical_log_lines(log) require(critical == process["critical_log_lines"], "Critical log report mismatch") text = Path(log).read_text(errors="replace") require(not re.search(r"Traceback|buffer\s+overflow|CUDA.*out.of.memory", text, re.I), "Failed runtime log") require("error" not in result and "shutdown_error" not in result, "Trial exception/shutdown failure") if expected_success: require(result["passed"] is True and result["status"] == "completed", "Failed required trial") require( process["passed"] is True and process["status"] == "completed" and process["returncode"] == 0 and not critical, "Failed required process", ) elif result["passed"]: require( process["passed"] and process["returncode"] == 0 and not critical, "Baseline process failed unexpectedly" ) else: require( result["status"] == "failed" and process["passed"] is False and process["returncode"] in (1, 2), "Invalid controlled failure", ) require( all( line.startswith("[ERROR] Command failed with code 1:") and "scripts/evaluate_standing.py" in line for line in critical ), "Unexpected critical failure", ) def memory_gate(samples): require([x["sim_seconds"] for x in samples] == list(range(1, 121)), "Missing stability memory samples") require(all(type(x["bytes"]) is int and x["bytes"] > 0 for x in samples), "Invalid memory measurement") first = max(x["bytes"] for x in samples if 2 <= x["sim_seconds"] <= 12) last = max(x["bytes"] for x in samples if 110 <= x["sim_seconds"] <= 120) return last / first - 1 def verify_trials(run, package, waiver_path=None, waiver_sha256=None): import torch from SkillFactory.legs.frozen_standing import FrozenStanding, checked_manifest, file_hash from standing_deviation import apply_drift_waiver, load_waiver from standing_protocol import inputs_for, reduce_trace torch.set_num_threads(4) lock = read_json(run / "candidate_lock.json") manifest_hash = file_hash(package / "manifest.json") manifest = checked_manifest(package, lock["manifest_sha256"]) require( lock["passed"] and lock["actor_extraction_max_error"] <= 1e-6 and lock["selected_update"] == 1024, "Invalid candidate extraction", ) require(manifest["files"] == lock["core_files"], "Candidate files changed") b = read_json(package / "binding.json") cfg = read_json(package / "task.json") policy = FrozenStanding( package, manifest_sha256=manifest_hash, task_sha256=b["task_sha256"], support_sha256=b["support_sha256"] ) independent = read_json(run / "independent_candidate.json") require( independent["passed"] and independent["status"] == "completed" and independent["isolated"] and independent["package_only"] and independent["manifest_sha256"] == manifest_hash and independent["actor_sha256"] == file_hash(package / "actor.pt") and independent["deterministic_batches"] == [1, 16, 64, 256] and independent["calls_per_batch"] == 100 and independent["invalid_inputs_binding_corruption_rejected"] and independent["input_state_unchanged"] and not independent["forbidden_imports"] and independent["requires_grad"] is False, "Independent candidate load failed", ) waiver = load_waiver(waiver_path, waiver_sha256, run, package) trials, artifacts, waiver_receipts = {}, {}, [] source_bundle = read_json(Path(cfg["source_bundle"]) / "manifest.json") expected_mass = read_json(Path("logs/loaded_stand/v0/standing_clean/result.json"))["total_mass_kg"] for name, mode, controller in ( ("full", "full", "policy"), ("holdout", "holdout", "policy"), ("hold", "holdout", "hold"), ("stability", "stability", "policy"), ("exploratory", "exploratory", "policy"), ): root = run / name report = read_json(root / "result.json") process = read_json(run / f"{name}_process.json") meta = read_json(root / "metadata.json") require( report["mode"] == mode and report["controller"] == controller and report["metadata"] == meta, "Trial identity/metadata mismatch", ) require(meta["inputs"] == inputs_for(mode), "Wrong holdout/full/stability protocol") require( report["package_manifest_sha256"] == manifest_hash and report["actor_sha256"] == file_hash(package / "actor.pt"), "Physics tested different policy identity", ) require(report["physics_trace_sha256"] == file_hash(root / "physics_trace.pt"), "Trace changed") require( abs(meta["total_mass_kg"] - expected_mass) < 1e-6 and abs(meta["height_m"] - b["height_m"]) < 1e-8, "Rig changed", ) require( meta["limits"] == [[j["limits"]["lower"], j["limits"]["upper"]] for j in source_bundle["joints"]] and meta["speeds"] == [j["limits"]["velocity"] for j in source_bundle["joints"]] and meta["efforts"] == torch.tensor([j["limits"]["effort"] for j in source_bundle["joints"]], dtype=torch.float32).tolist(), "Hardware limits changed", ) expected_sources = set(b["training_identity"]["sources"]) | { "scripts/evaluate_standing.py", "scripts/standing_protocol.py", "scripts/runtime_common.py", "scripts/supervise_sim.py", } require(set(report["source_hashes"]) == expected_sources, "Incomplete execution source identity") for path, digest in report["source_hashes"].items(): require( file_hash(path) == digest == file_hash(root / "source_snapshot" / path), "Evaluation source changed" ) data = torch.load(root / "physics_trace.pt", map_location="cpu", weights_only=True) reduced = reduce_trace(data, meta, cfg) del data require( report["physical_samples"] == reduced["physical_samples"] and report["rows"] == reduced["rows"], "Reduction mismatch", ) require(report["mean_height_error_m"] == reduced["mean_height_error_m"], "Metric mismatch") gates = reduced["gates"].copy() if mode == "full": for key in ("indexed_reset", "timeout_terminal_reset", "bad_action_atomic_rejection"): require(report["gates"].get(key) is True, f"Missing real reset/action probe: {key}") gates[key] = True if mode == "stability": growth = memory_gate(read_json(root / "memory.json")) require(growth == report["memory_growth_fraction"], "Memory report mismatch") gates["memory"] = growth <= 0.05 require(gates == report["gates"] and all(gates.values()) == report["passed"], "Gates/declared pass mismatch") required = name in ("full", "holdout", "stability") waived = name == "stability" and waiver is not None if waived: waiver_receipts.append(apply_drift_waiver(waiver, gates, reduced["rows"], cfg, meta)) # A waiver covers a metric, never backend errors, incomplete traces or bad cleanup. check_process(report, process, run / f"{name}.log", required and not waived) if required: require(all(gates.values()) or waived, f"Required physics failed: {name}") samples = torch.load(root / "inference_samples.pt", map_location="cpu", weights_only=True) if controller == "policy": with torch.inference_mode(): error = (policy.act(samples["observations"]) - samples["actions"]).abs().max().item() require(math.isfinite(error) and error <= 1e-5, f"CPU/CUDA inference mismatch: {error}") else: require((samples["actions"] == 0).all().item(), "Hold baseline is not zero-action") error = 0.0 trials[name] = { "passed": report["passed"], "accepted_by_operator_waiver": waived, "gates": gates, "mean_height_error_m": reduced["mean_height_error_m"], "cpu_cuda_max_error": error, "physical_samples": reduced["physical_samples"], "max_height_error_m": max(r["max_height_error_m"] for r in report["rows"]), "max_tilt_rad": max(r["max_tilt_rad"] for r in report["rows"]), "max_foot_drift_m": max(r["max_foot_drift_m"] for r in report["rows"]), "min_contact_fraction": min(r["contact_fraction"] for r in report["rows"]), "sensor_force_error_N": reduced["sensor_static_max_force_error_N"], "sensor_torque_error_Nm": reduced["sensor_static_max_torque_error_Nm"], "memory_growth_fraction": report.get("memory_growth_fraction"), } for p in ( root / "result.json", root / "metadata.json", root / "physics_trace.pt", root / "inference_samples.pt", root / "memory.json", run / f"{name}_process.json", run / f"{name}.log", ): artifacts[str(p.resolve())] = file_hash(p) if waiver is not None: require(len(waiver_receipts) == 1, "Waiver must apply exactly once") artifacts[str(Path(waiver_path).resolve())] = waiver_sha256 improvement = 1 - trials["holdout"]["mean_height_error_m"] / trials["hold"]["mean_height_error_m"] require(improvement >= 0.30, "Holdout improvement <30%") return { "trials": trials, "holdout_improvement_fraction": improvement, "artifacts": artifacts, "candidate_manifest_sha256": manifest_hash, "core_files": lock["core_files"], "acceptance_basis": "operator_waiver" if waiver is not None else "all_required_physics_gates", "unwaived_required_physics_passed": all(trials[k]["passed"] for k in ("full", "holdout", "stability")), "waiver": waiver, "waiver_sha256": waiver_sha256, "waiver_receipts": waiver_receipts, } def verify_training(package): from package_standing import SELECTED_SHA256 from SkillFactory.legs.frozen_standing import file_hash from SkillFactory.legs.standing_ppo import canonical_hash run = Path("logs/loaded_stand/train_v1") result = read_json(run / "result.json") config = read_json(run / "config.json") binding = read_json(package / "binding.json") require( result["formal_training_complete"] and result["passed"] and result["status"] == "completed" and result["update"] == config["updates"] == 2048 and result["transitions"] == 16777216, "Incomplete training", ) require( result["identity"] == binding["training_identity"] and result["identity"]["config"] == canonical_hash(config), "Training identity mismatch", ) require( binding["checkpoint_sha256"] == SELECTED_SHA256 == file_hash(run / "checkpoints/best_validation.pt"), "Selected checkpoint changed", ) rows = [json.loads(line) for line in (run / "metrics.jsonl").read_text().splitlines()] train = [r for r in rows if r["kind"] == "train"] require( [r["update"] for r in train] == list(range(1, 2049)) and all(r["transitions"] == r["update"] * 8192 for r in train), "Training sequence incomplete", ) values = [read_json(p) for p in sorted((run / "validation").glob("update_*.json"))] require([r["update"] for r in values] == list(range(256, 2049, 256)), "Missing validation") best = min(values, key=lambda r: (r["score"], r["update"])) require( best["update"] == 1024 and read_json(package / "selection.json")["score"] == best["score"], "Not validation-selected", ) require( all(r["input_hash"] == canonical_hash(r["inputs"]) == values[0]["input_hash"] for r in values), "Validation inputs changed", ) for path, digest in result["identity"]["sources"].items(): require(file_hash(path) == digest, "Training-bound source changed") check_process(result, read_json(run.parent / "train_v1_process.json"), run.parent / "train_v1.log", True) return {"updates": 2048, "transitions": 16777216, "selected_update": 1024, "checkpoint_sha256": SELECTED_SHA256} def audit(run, output, waiver_path=None, waiver_sha256=None, engineering_path=None): from SkillFactory.legs.frozen_standing import file_hash write_json(output, {"passed": False, "status": "running", "accepted_skill": False}) try: package = run / "candidate" training = verify_training(package) evidence = verify_trials(run, package, waiver_path, waiver_sha256) engineering_path = Path(engineering_path) if engineering_path else run / "engineering.json" engineering = read_json(engineering_path) require(engineering["passed"] is True and engineering["review_passed"] is True, "Engineering/review not passed") require( re.search(r"(?i)(?:merge verdict|verdict).{0,8}\bOK\b", Path(engineering["review_report"]).read_text()), "Missing positive final review verdict", ) require( { "scripts/standing_acceptance.py", "scripts/publish_standing.py", "scripts/verify_frozen_standing.py", "tests/test_standing_acceptance.py", engineering["review_report"], engineering["pytest_log"], engineering["ruff_log"], }.issubset(engineering["files"]), "Incomplete engineering identity", ) if waiver_path is not None: require("scripts/standing_deviation.py" in engineering["files"], "Missing waiver code review identity") for path, digest in engineering["files"].items(): require(file_hash(path) == digest, "Engineering evidence changed") require("All checks passed!" in Path(engineering["ruff_log"]).read_text(), "Ruff not passed") require( re.search(r"\d+ passed", Path(engineering["pytest_log"]).read_text()) and not re.search(r"\d+ failed|ERROR collecting", Path(engineering["pytest_log"]).read_text()), "Tests failed", ) root = run.parent wall = sum(read_json(p)["wall_seconds"] for p in root.rglob("*_process.json")) size = sum(p.stat().st_size for p in root.rglob("*") if p.is_file()) require(wall <= 1800 and size < 2 * 1024**3, "Acceptance resource budget exceeded") evidence.update( passed=True, status="completed", accepted_skill=False, stage="eligible_for_frozen_publication", training=training, physics_wall_seconds=wall, artifact_bytes=size, ) evidence["engineering_path"] = str(engineering_path.resolve()) for path in (engineering_path, run / "candidate_lock.json", run / "independent_candidate.json"): evidence["artifacts"][str(path.resolve())] = file_hash(path) write_json(output, evidence) return evidence except BaseException: write_json( output, {"passed": False, "status": "failed", "accepted_skill": False, "error": traceback.format_exc()} ) raise def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--run", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--waiver", type=Path) parser.add_argument("--waiver-sha256") parser.add_argument("--engineering", type=Path) args = parser.parse_args() audit(args.run, args.output, args.waiver, args.waiver_sha256, args.engineering) if __name__ == "__main__": main()