#!/usr/bin/env python3 """Fresh 46D residual PPO pilot. --prepare neither launches Kit nor authorizes learning. Only a parent-bound final manifest grant permits --train. No resume/warm-start CLI. The sole actual training launch must be supervised with a <=3480s outer timeout. """ import argparse import hashlib import json import math import signal import sys import traceback from pathlib import Path from runtime_common import critical_log_lines ROOT = Path(__file__).resolve().parents[1] def execute(env, cfg, args, manifest, ledger, guard): import torch from audit_contact_cycle_functional import evaluate from SkillFactory.legs.contact_cycle_sim.model import SimResidualActorCritic from SkillFactory.legs.contact_cycle_sim.ppo import ( features, save_checkpoint, selection_key, ) from SkillFactory.legs.contact_cycle_sim.training import ( MODEL_CONFIG, artifact_bytes, checkpoint_then_commit, choose_candidate, isolated_validation_rng, optimize_reserved_rollout, parameter_deltas, save_trace_subset, source_identity, validation_rows, verify_snapshot, ) from SkillFactory.tasks.manager_based.contact_cycle_sim.mdp import state from SkillFactory.tasks.manager_based.contact_cycle_sim.recording import SimCycleRecorder torch.set_num_threads(4) s = state(env) if (s.robot.is_fixed_base or env.cfg.sim.physics.solver_type != 0 or env.cfg.sim.physics.enable_external_forces_every_iteration or env.cfg.terminations.failed is None or env.physics_dt != .002 or env.step_dt != .01 or env.num_envs != 128 or env.cfg.episode_length_s != 9): raise RuntimeError("Require unchanged floating-root PGS, active failures, 500Hz physics and 9s nominal task") runtime = {"torch": str(torch.__version__), "device": str(env.device), "physics_dt": env.physics_dt, "control_dt": env.step_dt, "num_envs": env.num_envs} identity = {"manifest": manifest, "runtime": runtime, "seed": cfg["seeds"][0], "model_config": MODEL_CONFIG} ledger.write_json(args.output / "runtime_identity.json", identity) functional = json.loads((ROOT / cfg["functional_config"]).read_text()) algo = {**cfg["algorithm"], "horizon": cfg["horizon"]} seed = cfg["seeds"][0] torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) model = SimResidualActorCritic(**MODEL_CONFIG).to(env.device) initial = {k: v.detach().cpu().clone() for k, v in model.named_parameters()} optimizer = torch.optim.Adam(model.parameters(), lr=algo["learning_rate"], eps=algo["adam_eps"]) best = None def check_source(): guard() if source_identity(ROOT, cfg, args.config) != manifest["sources"]: raise RuntimeError("Bound sources/assets/config changed; refusing optimization or publication") verify_snapshot(args.output, manifest) if critical_log_lines(args.console_log): raise RuntimeError("Critical simulator console log") def save(path, update): check_source() remaining = cfg["max_artifact_bytes"] - artifact_bytes(args.output) - 1048576 size = save_checkpoint(path, model, optimizer, identity, update, best, ledger.data, MODEL_CONFIG, max_bytes=remaining) ledger.guard() return size @torch.no_grad() def validation(update): check_source() with isolated_validation_rng(env, cfg["validation_seed"], seed): observation = features(env.reset()[0]["policy"]) if bool(s.scenarios.any() or s.stand_only.any()): raise RuntimeError("Validation must contain only nominal complete cycles") returns = torch.zeros(env.num_envs, device=env.device) live = torch.ones(env.num_envs, dtype=torch.bool, device=env.device) with SimCycleRecorder(env) as recorder: for step in range(900): guard() action, mean, _, _ = model.act(observation, deterministic=True) recorder.before_control(observation, action, mean, model.log_std.clamp(math.log(.02), math.log(.5)).exp()) obs, reward, terminated, truncated, _ = env.step(action) if not torch.isfinite(reward).all(): raise RuntimeError("Nonfinite validation reward") returns += reward * live live &= ~(terminated | truncated) recorder.end_control(terminated, truncated, step == 899) observation = features(obs["policy"]) trace = {k: torch.stack([frame[k] for frame in recorder.frames]) for k in recorder.frames[0]} # Recorded buffers are copies; release the original per-frame storage before serialization. recorder.frames.clear() rows = validation_rows(trace, recorder.stops, recorder.early, s.nominal, functional, evaluate) for row, total in zip(rows, returns.cpu().tolist()): row["episode_return"] = total result = {"kind": ("untrained_zero_residual_network" if update == 0 else "analytic_reference_plus_learned_residual"), "trained": update > 0, "update": update, "seed": seed, "validation_seed": cfg["validation_seed"], "rows": rows, "selection_key": list(selection_key(rows)), "parameter_deltas": parameter_deltas(model, initial), "accepted_skill": False, "independent_test_executed": False, "audit_controller_kind": "learned_policy_metadata_interface_only"} label = f"validation_u{update:04d}" ledger.write_json(args.output / f"{label}.json", result) selected = ([r["env"] for r in rows if not r["success"]] or list(range(env.num_envs)))[:4] def reserve(size): ledger.guard(reserve_bytes=size + 1048576) save_trace_subset(args.output / f"{label}_trace.pt", trace, selected, recorder.stops, reserve) save_trace_subset(args.output / f"{label}_initial.pt", {k: v[None] for k, v in recorder.initial_state.items()}, selected, [1] * env.num_envs, reserve) check_source() return result save(args.output / "untrained_u0000.pt", 0) baseline = validation(0) if not all(r["success"] for r in baseline["rows"]) or len(baseline["rows"]) != 128: raise RuntimeError("u0 did not pass 128/128 functional cycles; stop before the first training rollout") comparisons = [] x = features(env.reset()[0]["policy"]) for update in range(1, cfg["updates_per_seed"] + 1): x, failures, timeouts, learning = optimize_reserved_rollout( env, model, optimizer, x, algo, seed, update, ledger, guard, check_source) if update in cfg["validation_updates"]: row = validation(update) if choose_candidate(best, update, row["rows"]): name = f"candidate_u{update:04d}.pt" best = {"update": update, "selection_key": row["selection_key"], "path": name} save(args.output / name, update) best["sha256"] = hashlib.sha256((args.output / name).read_bytes()).hexdigest() comparisons.append({"update": update, "selection_key": row["selection_key"], "parameter_deltas": row["parameter_deltas"], "mean_height_rms_m": sum(r["height_rms_m"] for r in row["rows"]) / 128, "mean_return": sum(r["episode_return"] for r in row["rows"]) / 128, "mean_actual_residual_rms_rad": sum( r["actual_residual_rms_rad"] for r in row["rows"]) / 128}) x = features(env.reset()[0]["policy"]) # Pending reservation remains charged on ANY failure; no automatic recovery. latest_bytes = checkpoint_then_commit( ledger, seed, update, lambda: save(args.output / "latest.pt", update)) ledger.append_json(args.output / "metrics.jsonl", {"update": update, "learner": learning, "failures": failures, "timeouts": timeouts, "latest_checkpoint_bytes": latest_bytes, "reserved_transitions": ledger.data["reserved_transitions"]}) check_source() if (best is None or best["update"] == 0 or hashlib.sha256((args.output / best["path"]).read_bytes()).hexdigest() != best["sha256"]): raise RuntimeError("Missing or changed trained candidate") locked = {"candidate": best, "accepted_skill": False, "independent_test_executed": False, "training_scope": "nominal_simulation_residual_pilot", "rule": "failure fraction then all-row cost; earliest tie; never reselect from later outcomes"} ledger.write_json(args.output / "locked_candidate.json", locked) ledger.write_json(args.output / "baseline_comparison.json", { "baseline_kind": baseline["kind"], "baseline_selection_key": baseline["selection_key"], "baseline_mean_height_rms_m": sum(r["height_rms_m"] for r in baseline["rows"]) / 128, "baseline_mean_return": sum(r["episode_return"] for r in baseline["rows"]) / 128, "baseline_mean_actual_residual_rms_rad": sum(r["actual_residual_rms_rad"] for r in baseline["rows"]) / 128, "updated_models": comparisons, "selection_changed_by_comparison": False, "note": "Reference already passed at u0. Nonzero parameter deltas alone do not demonstrate improvement.", "accepted_skill": False}) return {"passed": True, "training_complete": True, "accepted_skill": False, "locked": locked} def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--prepare", action="store_true") mode.add_argument("--train", action="store_true") parser.add_argument("--config", type=Path, default=ROOT / "configs/contact_cycle/sim_training_v1.json") parser.add_argument("--output", type=Path, required=True) parser.add_argument("--authorization", type=Path) parser.add_argument("--console-log", type=Path) parser.add_argument("--supervisor-report", type=Path) parser.add_argument("--headless", action="store_true") parser.add_argument("--device", default="cuda:0") args = parser.parse_args(argv) from SkillFactory.legs.contact_cycle_sim.training import ( FreshBudgetLedger, assert_no_conflicting_processes, make_manifest, prepare, require_authorization, validate_prepared, writable_output, ) args.output = writable_output(ROOT, args.output) if args.prepare: if args.authorization is not None or args.console_log is not None or args.supervisor_report is not None: parser.error("Preparation is not authorization or a supervised launch") prepare(ROOT, args.config, args.output) print(f"Prepared only; no Kit, budget or optimizer: {args.output}", flush=True) return # Every gate here runs BEFORE importing Kit and BEFORE the ledger clock starts. manifest = make_manifest(ROOT, args.config) grant = require_authorization(args.authorization, manifest) validate_prepared(ROOT, args.output, manifest) if not args.headless: raise ValueError("Pilot must be headless and externally supervised") if (args.console_log is None or args.console_log.resolve() != args.output / "console.log" or args.supervisor_report is None or args.supervisor_report.resolve() != args.output / "supervisor.json" or not args.console_log.is_file() or not args.supervisor_report.is_file()): raise ValueError("Require live supervisor console.log and supervisor.json inside artifact budget") supervisor = json.loads(args.supervisor_report.read_text()) if supervisor.get("status") != "running" or supervisor.get("passed") is not False: raise ValueError("Require a running external supervisor") assert_no_conflicting_processes() cfg = manifest["training"] stop = {"requested": False} for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, lambda *_: stop.update(requested=True)) app = env = safety = None result = {"passed": False, "training_complete": False, "accepted_skill": False} code = 1 with FreshBudgetLedger(args.output, manifest) as ledger: def guard(): ledger.guard() if stop["requested"]: raise InterruptedError("Cooperative stop; pending work remains charged; no retry/resume") if safety is not None: safety.check() ledger.write_json(args.output / "authorization_record.json", grant) try: guard() from isaaclab.app import AppLauncher app = AppLauncher(headless=True, device=args.device).app guard() from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit from SkillFactory.tasks.manager_based.contact_cycle_sim.cfg import SimContactCycleEnvCfg from SkillFactory.tasks.manager_based.contact_cycle_sim.env import SimContactCycleEnv environment_cfg = SimContactCycleEnvCfg(residual_config=str(ROOT / cfg["task_config"])) environment_cfg.scene.num_envs = cfg["num_envs"] environment_cfg.sim.device = args.device environment_cfg.seed = cfg["seeds"][0] env = SimContactCycleEnv(cfg=environment_cfg) thresholds = json.loads((ROOT / cfg["safety_config"]).read_text())["gates"] with PhysicalSafetyAudit(env, thresholds) as safety: result = execute(env, cfg, args, manifest, ledger, guard) guard() code = 0 except BaseException: result.update(passed=False, error=traceback.format_exc()) traceback.print_exc() finally: if env is not None: try: env.close() except BaseException: result.update(passed=False, shutdown_error=traceback.format_exc()) code = 1 result["budget"] = ledger.data if safety is not None: result["physics_safety"] = safety.report() try: guard() if critical_log_lines(args.console_log): raise RuntimeError("Critical console log during shutdown") except BaseException: result.update(passed=False, final_guard_error=traceback.format_exc()) code = 1 result["status"] = "completed" if code == 0 else "failed" # Never bypass the artifact/deadline guard to publish a success on failure. try: ledger.write_json(args.output / "result.json", result) finally: if app is not None: app.close(exit_code=code) if code: sys.exit(code) if __name__ == "__main__": main()