093ab883be
Add loaded-standing physics, PPO training, fixed-candidate acceptance, isolated inference, and fail-closed publication. Preserve the original 120-second drift failure and bind the operator-approved exception to the exact candidate and evidence. Include pending Reach training/release infrastructure and archived packages. Leave pre-existing short-worker visualization changes and large generated training artifacts out of this commit.
355 lines
16 KiB
Python
355 lines
16 KiB
Python
"""Fail-closed standing evidence audit. Re-read traces and exercise CPU runtime; no simulation."""
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import re
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
from runtime_common import critical_log_lines, write_json
|
|
|
|
|
|
def read_json(path):
|
|
def invalid(value):
|
|
raise ValueError(f"Nonfinite JSON: {value}")
|
|
|
|
return json.loads(Path(path).read_text(), parse_constant=invalid)
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def check_process(result, process, log, expected_success):
|
|
require(
|
|
process.get("cleanup", {}).get("clean") is True and process["cleanup"]["remaining"] == [], "Unclean process"
|
|
)
|
|
critical = critical_log_lines(log)
|
|
require(critical == process["critical_log_lines"], "Critical log report mismatch")
|
|
text = Path(log).read_text(errors="replace")
|
|
require(not re.search(r"Traceback|buffer\s+overflow|CUDA.*out.of.memory", text, re.I), "Failed runtime log")
|
|
require("error" not in result and "shutdown_error" not in result, "Trial exception/shutdown failure")
|
|
if expected_success:
|
|
require(result["passed"] is True and result["status"] == "completed", "Failed required trial")
|
|
require(
|
|
process["passed"] is True
|
|
and process["status"] == "completed"
|
|
and process["returncode"] == 0
|
|
and not critical,
|
|
"Failed required process",
|
|
)
|
|
elif result["passed"]:
|
|
require(
|
|
process["passed"] and process["returncode"] == 0 and not critical, "Baseline process failed unexpectedly"
|
|
)
|
|
else:
|
|
require(
|
|
result["status"] == "failed" and process["passed"] is False and process["returncode"] in (1, 2),
|
|
"Invalid controlled failure",
|
|
)
|
|
require(
|
|
all(
|
|
line.startswith("[ERROR] Command failed with code 1:") and "scripts/evaluate_standing.py" in line
|
|
for line in critical
|
|
),
|
|
"Unexpected critical failure",
|
|
)
|
|
|
|
|
|
def memory_gate(samples):
|
|
require([x["sim_seconds"] for x in samples] == list(range(1, 121)), "Missing stability memory samples")
|
|
require(all(type(x["bytes"]) is int and x["bytes"] > 0 for x in samples), "Invalid memory measurement")
|
|
first = max(x["bytes"] for x in samples if 2 <= x["sim_seconds"] <= 12)
|
|
last = max(x["bytes"] for x in samples if 110 <= x["sim_seconds"] <= 120)
|
|
return last / first - 1
|
|
|
|
|
|
def verify_trials(run, package, waiver_path=None, waiver_sha256=None):
|
|
import torch
|
|
from SkillFactory.legs.frozen_standing import FrozenStanding, checked_manifest, file_hash
|
|
from standing_deviation import apply_drift_waiver, load_waiver
|
|
from standing_protocol import inputs_for, reduce_trace
|
|
|
|
torch.set_num_threads(4)
|
|
lock = read_json(run / "candidate_lock.json")
|
|
manifest_hash = file_hash(package / "manifest.json")
|
|
manifest = checked_manifest(package, lock["manifest_sha256"])
|
|
require(
|
|
lock["passed"] and lock["actor_extraction_max_error"] <= 1e-6 and lock["selected_update"] == 1024,
|
|
"Invalid candidate extraction",
|
|
)
|
|
require(manifest["files"] == lock["core_files"], "Candidate files changed")
|
|
b = read_json(package / "binding.json")
|
|
cfg = read_json(package / "task.json")
|
|
policy = FrozenStanding(
|
|
package, manifest_sha256=manifest_hash, task_sha256=b["task_sha256"], support_sha256=b["support_sha256"]
|
|
)
|
|
independent = read_json(run / "independent_candidate.json")
|
|
require(
|
|
independent["passed"]
|
|
and independent["status"] == "completed"
|
|
and independent["isolated"]
|
|
and independent["package_only"]
|
|
and independent["manifest_sha256"] == manifest_hash
|
|
and independent["actor_sha256"] == file_hash(package / "actor.pt")
|
|
and independent["deterministic_batches"] == [1, 16, 64, 256]
|
|
and independent["calls_per_batch"] == 100
|
|
and independent["invalid_inputs_binding_corruption_rejected"]
|
|
and independent["input_state_unchanged"]
|
|
and not independent["forbidden_imports"]
|
|
and independent["requires_grad"] is False,
|
|
"Independent candidate load failed",
|
|
)
|
|
waiver = load_waiver(waiver_path, waiver_sha256, run, package)
|
|
trials, artifacts, waiver_receipts = {}, {}, []
|
|
source_bundle = read_json(Path(cfg["source_bundle"]) / "manifest.json")
|
|
expected_mass = read_json(Path("logs/loaded_stand/v0/standing_clean/result.json"))["total_mass_kg"]
|
|
for name, mode, controller in (
|
|
("full", "full", "policy"),
|
|
("holdout", "holdout", "policy"),
|
|
("hold", "holdout", "hold"),
|
|
("stability", "stability", "policy"),
|
|
("exploratory", "exploratory", "policy"),
|
|
):
|
|
root = run / name
|
|
report = read_json(root / "result.json")
|
|
process = read_json(run / f"{name}_process.json")
|
|
meta = read_json(root / "metadata.json")
|
|
require(
|
|
report["mode"] == mode and report["controller"] == controller and report["metadata"] == meta,
|
|
"Trial identity/metadata mismatch",
|
|
)
|
|
require(meta["inputs"] == inputs_for(mode), "Wrong holdout/full/stability protocol")
|
|
require(
|
|
report["package_manifest_sha256"] == manifest_hash
|
|
and report["actor_sha256"] == file_hash(package / "actor.pt"),
|
|
"Physics tested different policy identity",
|
|
)
|
|
require(report["physics_trace_sha256"] == file_hash(root / "physics_trace.pt"), "Trace changed")
|
|
require(
|
|
abs(meta["total_mass_kg"] - expected_mass) < 1e-6 and abs(meta["height_m"] - b["height_m"]) < 1e-8,
|
|
"Rig changed",
|
|
)
|
|
require(
|
|
meta["limits"] == [[j["limits"]["lower"], j["limits"]["upper"]] for j in source_bundle["joints"]]
|
|
and meta["speeds"] == [j["limits"]["velocity"] for j in source_bundle["joints"]]
|
|
and meta["efforts"]
|
|
== torch.tensor([j["limits"]["effort"] for j in source_bundle["joints"]], dtype=torch.float32).tolist(),
|
|
"Hardware limits changed",
|
|
)
|
|
expected_sources = set(b["training_identity"]["sources"]) | {
|
|
"scripts/evaluate_standing.py",
|
|
"scripts/standing_protocol.py",
|
|
"scripts/runtime_common.py",
|
|
"scripts/supervise_sim.py",
|
|
}
|
|
require(set(report["source_hashes"]) == expected_sources, "Incomplete execution source identity")
|
|
for path, digest in report["source_hashes"].items():
|
|
require(
|
|
file_hash(path) == digest == file_hash(root / "source_snapshot" / path), "Evaluation source changed"
|
|
)
|
|
data = torch.load(root / "physics_trace.pt", map_location="cpu", weights_only=True)
|
|
reduced = reduce_trace(data, meta, cfg)
|
|
del data
|
|
require(
|
|
report["physical_samples"] == reduced["physical_samples"] and report["rows"] == reduced["rows"],
|
|
"Reduction mismatch",
|
|
)
|
|
require(report["mean_height_error_m"] == reduced["mean_height_error_m"], "Metric mismatch")
|
|
gates = reduced["gates"].copy()
|
|
if mode == "full":
|
|
for key in ("indexed_reset", "timeout_terminal_reset", "bad_action_atomic_rejection"):
|
|
require(report["gates"].get(key) is True, f"Missing real reset/action probe: {key}")
|
|
gates[key] = True
|
|
if mode == "stability":
|
|
growth = memory_gate(read_json(root / "memory.json"))
|
|
require(growth == report["memory_growth_fraction"], "Memory report mismatch")
|
|
gates["memory"] = growth <= 0.05
|
|
require(gates == report["gates"] and all(gates.values()) == report["passed"], "Gates/declared pass mismatch")
|
|
required = name in ("full", "holdout", "stability")
|
|
waived = name == "stability" and waiver is not None
|
|
if waived:
|
|
waiver_receipts.append(apply_drift_waiver(waiver, gates, reduced["rows"], cfg, meta))
|
|
# A waiver covers a metric, never backend errors, incomplete traces or bad cleanup.
|
|
check_process(report, process, run / f"{name}.log", required and not waived)
|
|
if required:
|
|
require(all(gates.values()) or waived, f"Required physics failed: {name}")
|
|
samples = torch.load(root / "inference_samples.pt", map_location="cpu", weights_only=True)
|
|
if controller == "policy":
|
|
with torch.inference_mode():
|
|
error = (policy.act(samples["observations"]) - samples["actions"]).abs().max().item()
|
|
require(math.isfinite(error) and error <= 1e-5, f"CPU/CUDA inference mismatch: {error}")
|
|
else:
|
|
require((samples["actions"] == 0).all().item(), "Hold baseline is not zero-action")
|
|
error = 0.0
|
|
trials[name] = {
|
|
"passed": report["passed"],
|
|
"accepted_by_operator_waiver": waived,
|
|
"gates": gates,
|
|
"mean_height_error_m": reduced["mean_height_error_m"],
|
|
"cpu_cuda_max_error": error,
|
|
"physical_samples": reduced["physical_samples"],
|
|
"max_height_error_m": max(r["max_height_error_m"] for r in report["rows"]),
|
|
"max_tilt_rad": max(r["max_tilt_rad"] for r in report["rows"]),
|
|
"max_foot_drift_m": max(r["max_foot_drift_m"] for r in report["rows"]),
|
|
"min_contact_fraction": min(r["contact_fraction"] for r in report["rows"]),
|
|
"sensor_force_error_N": reduced["sensor_static_max_force_error_N"],
|
|
"sensor_torque_error_Nm": reduced["sensor_static_max_torque_error_Nm"],
|
|
"memory_growth_fraction": report.get("memory_growth_fraction"),
|
|
}
|
|
for p in (
|
|
root / "result.json",
|
|
root / "metadata.json",
|
|
root / "physics_trace.pt",
|
|
root / "inference_samples.pt",
|
|
root / "memory.json",
|
|
run / f"{name}_process.json",
|
|
run / f"{name}.log",
|
|
):
|
|
artifacts[str(p.resolve())] = file_hash(p)
|
|
if waiver is not None:
|
|
require(len(waiver_receipts) == 1, "Waiver must apply exactly once")
|
|
artifacts[str(Path(waiver_path).resolve())] = waiver_sha256
|
|
improvement = 1 - trials["holdout"]["mean_height_error_m"] / trials["hold"]["mean_height_error_m"]
|
|
require(improvement >= 0.30, "Holdout improvement <30%")
|
|
return {
|
|
"trials": trials,
|
|
"holdout_improvement_fraction": improvement,
|
|
"artifacts": artifacts,
|
|
"candidate_manifest_sha256": manifest_hash,
|
|
"core_files": lock["core_files"],
|
|
"acceptance_basis": "operator_waiver" if waiver is not None else "all_required_physics_gates",
|
|
"unwaived_required_physics_passed": all(trials[k]["passed"] for k in ("full", "holdout", "stability")),
|
|
"waiver": waiver,
|
|
"waiver_sha256": waiver_sha256,
|
|
"waiver_receipts": waiver_receipts,
|
|
}
|
|
|
|
|
|
def verify_training(package):
|
|
from package_standing import SELECTED_SHA256
|
|
from SkillFactory.legs.frozen_standing import file_hash
|
|
from SkillFactory.legs.standing_ppo import canonical_hash
|
|
|
|
run = Path("logs/loaded_stand/train_v1")
|
|
result = read_json(run / "result.json")
|
|
config = read_json(run / "config.json")
|
|
binding = read_json(package / "binding.json")
|
|
require(
|
|
result["formal_training_complete"]
|
|
and result["passed"]
|
|
and result["status"] == "completed"
|
|
and result["update"] == config["updates"] == 2048
|
|
and result["transitions"] == 16777216,
|
|
"Incomplete training",
|
|
)
|
|
require(
|
|
result["identity"] == binding["training_identity"] and result["identity"]["config"] == canonical_hash(config),
|
|
"Training identity mismatch",
|
|
)
|
|
require(
|
|
binding["checkpoint_sha256"] == SELECTED_SHA256 == file_hash(run / "checkpoints/best_validation.pt"),
|
|
"Selected checkpoint changed",
|
|
)
|
|
rows = [json.loads(line) for line in (run / "metrics.jsonl").read_text().splitlines()]
|
|
train = [r for r in rows if r["kind"] == "train"]
|
|
require(
|
|
[r["update"] for r in train] == list(range(1, 2049))
|
|
and all(r["transitions"] == r["update"] * 8192 for r in train),
|
|
"Training sequence incomplete",
|
|
)
|
|
values = [read_json(p) for p in sorted((run / "validation").glob("update_*.json"))]
|
|
require([r["update"] for r in values] == list(range(256, 2049, 256)), "Missing validation")
|
|
best = min(values, key=lambda r: (r["score"], r["update"]))
|
|
require(
|
|
best["update"] == 1024 and read_json(package / "selection.json")["score"] == best["score"],
|
|
"Not validation-selected",
|
|
)
|
|
require(
|
|
all(r["input_hash"] == canonical_hash(r["inputs"]) == values[0]["input_hash"] for r in values),
|
|
"Validation inputs changed",
|
|
)
|
|
for path, digest in result["identity"]["sources"].items():
|
|
require(file_hash(path) == digest, "Training-bound source changed")
|
|
check_process(result, read_json(run.parent / "train_v1_process.json"), run.parent / "train_v1.log", True)
|
|
return {"updates": 2048, "transitions": 16777216, "selected_update": 1024, "checkpoint_sha256": SELECTED_SHA256}
|
|
|
|
|
|
def audit(run, output, waiver_path=None, waiver_sha256=None, engineering_path=None):
|
|
from SkillFactory.legs.frozen_standing import file_hash
|
|
|
|
write_json(output, {"passed": False, "status": "running", "accepted_skill": False})
|
|
try:
|
|
package = run / "candidate"
|
|
training = verify_training(package)
|
|
evidence = verify_trials(run, package, waiver_path, waiver_sha256)
|
|
engineering_path = Path(engineering_path) if engineering_path else run / "engineering.json"
|
|
engineering = read_json(engineering_path)
|
|
require(engineering["passed"] is True and engineering["review_passed"] is True, "Engineering/review not passed")
|
|
require(
|
|
re.search(r"(?i)(?:merge verdict|verdict).{0,8}\bOK\b", Path(engineering["review_report"]).read_text()),
|
|
"Missing positive final review verdict",
|
|
)
|
|
require(
|
|
{
|
|
"scripts/standing_acceptance.py",
|
|
"scripts/publish_standing.py",
|
|
"scripts/verify_frozen_standing.py",
|
|
"tests/test_standing_acceptance.py",
|
|
engineering["review_report"],
|
|
engineering["pytest_log"],
|
|
engineering["ruff_log"],
|
|
}.issubset(engineering["files"]),
|
|
"Incomplete engineering identity",
|
|
)
|
|
if waiver_path is not None:
|
|
require("scripts/standing_deviation.py" in engineering["files"], "Missing waiver code review identity")
|
|
for path, digest in engineering["files"].items():
|
|
require(file_hash(path) == digest, "Engineering evidence changed")
|
|
require("All checks passed!" in Path(engineering["ruff_log"]).read_text(), "Ruff not passed")
|
|
require(
|
|
re.search(r"\d+ passed", Path(engineering["pytest_log"]).read_text())
|
|
and not re.search(r"\d+ failed|ERROR collecting", Path(engineering["pytest_log"]).read_text()),
|
|
"Tests failed",
|
|
)
|
|
root = run.parent
|
|
wall = sum(read_json(p)["wall_seconds"] for p in root.rglob("*_process.json"))
|
|
size = sum(p.stat().st_size for p in root.rglob("*") if p.is_file())
|
|
require(wall <= 1800 and size < 2 * 1024**3, "Acceptance resource budget exceeded")
|
|
evidence.update(
|
|
passed=True,
|
|
status="completed",
|
|
accepted_skill=False,
|
|
stage="eligible_for_frozen_publication",
|
|
training=training,
|
|
physics_wall_seconds=wall,
|
|
artifact_bytes=size,
|
|
)
|
|
evidence["engineering_path"] = str(engineering_path.resolve())
|
|
for path in (engineering_path, run / "candidate_lock.json", run / "independent_candidate.json"):
|
|
evidence["artifacts"][str(path.resolve())] = file_hash(path)
|
|
write_json(output, evidence)
|
|
return evidence
|
|
except BaseException:
|
|
write_json(
|
|
output, {"passed": False, "status": "failed", "accepted_skill": False, "error": traceback.format_exc()}
|
|
)
|
|
raise
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--run", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--waiver", type=Path)
|
|
parser.add_argument("--waiver-sha256")
|
|
parser.add_argument("--engineering", type=Path)
|
|
args = parser.parse_args()
|
|
audit(args.run, args.output, args.waiver, args.waiver_sha256, args.engineering)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|