#!/usr/bin/env python3 """Explicitly authorized contact-cycle PPO. --prepare does not launch Kit or train. Independent testing is intentionally not an option. A continuation request or a previous standing authorization is NOT a grant for this runner. """ import argparse import hashlib import json import math import shutil import signal import sys import traceback from pathlib import Path from runtime_common import critical_log_lines, finish_run, write_json ROOT = Path(__file__).resolve().parents[1] def execute(env, cfg, args, manifest, ledger, safety): import torch from SkillFactory.legs.contact_cycle.curriculum import EpisodeSampler from SkillFactory.legs.contact_cycle.ppo import ( CycleActorCritic, collect_rollout, features, ppo_step, restore_checkpoint, restore_rng, rng_state, rollout_diagnostics, save_checkpoint, selection_score, ) from SkillFactory.legs.contact_cycle.training import artifact_bytes, save_trace_subset, source_identity from SkillFactory.tasks.manager_based.contact_cycle.mdp import state from SkillFactory.tasks.manager_based.contact_cycle.recording import CycleRecorder torch.set_num_threads(4) s = state(env) if (s.robot.is_fixed_base or env.cfg.sim.physics.solver_type != 0 or env.cfg.sim.physics.enable_external_forces_every_iteration): raise RuntimeError("Require validated floating-root PGS/external-force physics settings") stop = {"requested": False} signal.signal(signal.SIGTERM, lambda *_: stop.update(requested=True)) signal.signal(signal.SIGINT, lambda *_: stop.update(requested=True)) runtime = {"torch": str(torch.__version__), "device": str(env.device), "physics_dt": env.physics_dt, "control_dt": env.step_dt, "num_envs": env.num_envs} identity = {"manifest": manifest, "runtime": runtime} runtime_path = args.output/"runtime_identity.json" if runtime_path.exists() and json.loads(runtime_path.read_text()) != runtime: raise RuntimeError("Runtime identity changed across resume") ledger.write_json(runtime_path, runtime) thresholds = json.loads((ROOT/cfg["proposal"]).read_text())["gates"] algo = {**cfg["algorithm"], "horizon": cfg["horizon"]} def guard(): ledger.guard() safety.check() if stop["requested"]: raise InterruptedError("Operator requested stop; resume from last atomic learner checkpoint") def check_source(): guard() if source_identity(ROOT, cfg, args.config) != manifest["sources"]: raise RuntimeError("Training inputs changed; refusing continuation/checkpoint") if args.console_log is not None and critical_log_lines(args.console_log): raise RuntimeError("Critical runtime log; refusing successful completion") def save(path, model, optimizer, seed_identity, update, sampler, best): guard() remaining = cfg["max_artifact_bytes"] - artifact_bytes(args.output) - 1024*1024 save_checkpoint(path, model, optimizer, seed_identity, update, sampler, best, ledger.data, max_bytes=remaining) ledger.guard() candidates = [] for seed in cfg["seeds"]: guard() seed_dir = args.output/f"seed_{seed}" seed_dir.mkdir(exist_ok=True) seed_identity = {**identity, "seed": seed} torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) model = CycleActorCritic(cfg["algorithm"]["hidden"], cfg["algorithm"]["initial_std"]).to(env.device) optimizer = torch.optim.Adam(model.parameters(), lr=cfg["algorithm"]["learning_rate"], eps=1e-5) sampler = EpisodeSampler(s.cc, env.num_envs, seed) latest = seed_dir/"latest.pt" update, best = 0, None if latest.exists(): checkpoint = restore_checkpoint(latest, model, optimizer, seed_identity, sampler) update, best = checkpoint["update"], checkpoint["best"] if ledger.data["pending"] is not None and ledger.data["pending"]["seed"] != seed: if update != cfg["updates_per_seed"]: raise RuntimeError("Unexpected unfinished earlier seed") else: ledger.reconcile_checkpoint(seed, update) s.set_sampler(sampler) x = features(env.reset()[0]["policy"]) if not latest.exists(): save(latest, model, optimizer, seed_identity, 0, sampler, None) def log(row): ledger.append_json(seed_dir/"metrics.jsonl", row) @torch.no_grad() def validation(): check_source() saved_rng = rng_state() validation_sampler = EpisodeSampler(s.cc, env.num_envs, cfg["validation_seed"], split="validation") s.set_sampler(validation_sampler) observation = env.reset()[0]["policy"] inputs = {"scenario": s.scenarios.cpu().tolist(), "commands": s.schedule.commands.cpu().tolist(), "points": s.sampled_points.cpu().tolist(), "delays": s.schedule.delays.cpu().tolist()} if set(inputs["scenario"]) != set(range(4)): raise RuntimeError("Validation family coverage missing") steps = round(sum(s.cc["phase_durations_s"])/env.step_dt) try: with CycleRecorder(env) as recorder: for step in range(steps): guard() action, mean, _, _ = model.act(features(observation), deterministic=True) recorder.before_control(observation, action, mean, model.log_std.clamp(math.log(0.02), math.log(0.5)).exp()) obs, _, terminated, truncated, _ = env.step(action) recorder.end_control(terminated, truncated, step == steps - 1) observation = obs["policy"] rows, trace = recorder.summarize(thresholds) score = selection_score(rows) row = {"kind": "validation_only", "seed": seed, "validation_seed": cfg["validation_seed"], "update": update, "reservation": ledger.data["reservations"], "score": score, "failure_fraction": sum(not r["success"] for r in rows)/len(rows), "inputs": inputs, "rows": rows, "accepted_skill": False} label = f"validation_u{update:04d}_r{ledger.data['reservations']:06d}" ledger.write_json(seed_dir/f"{label}.json", row) # Keep bounded real failure traces: first four failed first episodes (or first four successes). selected = ([r["env"] for r in rows if not r["success"]] or list(range(env.num_envs)))[:4] save_trace_subset(seed_dir/f"{label}_trace.pt", trace, selected, recorder.stops, lambda size: ledger.guard(reserve_bytes=size + 1024*1024)) save_trace_subset(seed_dir/f"{label}_initial.pt", {k: v[None] for k, v in recorder.initial_state.items()}, selected, [1]*env.num_envs, lambda size: ledger.guard(reserve_bytes=size + 1024*1024)) ledger.guard() return row finally: s.set_sampler(sampler) restore_rng(saved_rng) while update < cfg["updates_per_seed"]: check_source() sampler.set_update(update + 1) ledger.reserve_rollout(seed, update + 1) data, x, failures, timeouts = collect_rollout(env, model, x, algo, guard) guard() diagnostic_row = rollout_diagnostics(data, algo) if diagnostic_row["missing_diagnostic_samples"]: raise RuntimeError("Missing control-clock diagnostics; refusing optimizer update") learning = ppo_step(model, optimizer, data, algo) update += 1 if update % cfg["validation_interval"] == 0: validation_row = validation() log(validation_row) if best is None or validation_row["score"] < best["score"]: relative = f"candidate_u{update:04d}_r{ledger.data['reservations']:06d}.pt" best = {"update": update, "score": validation_row["score"], "path": relative, "failure_fraction": validation_row["failure_fraction"]} save(seed_dir/relative, model, optimizer, seed_identity, update, sampler, best) best["sha256"] = hashlib.sha256((seed_dir/relative).read_bytes()).hexdigest() x = features(env.reset()[0]["policy"]) # Checkpoint BEFORE durable progress commit; ledger reconciles the crash window on resume. check_source() save(latest, model, optimizer, seed_identity, update, sampler, best) ledger.commit_update(seed, update) log({"kind": "training", "update": update, "seed": seed, "learner": learning, "terminated": failures, "timeouts": timeouts, "curriculum_stage": sampler.stage, "episode_counts": sampler.episodes.tolist(), "diagnostics": diagnostic_row, "reserved_transitions": ledger.data["reserved_transitions"]}) if best is None or hashlib.sha256((seed_dir/best["path"]).read_bytes()).hexdigest() != best["sha256"]: raise RuntimeError("Missing/changed fixed candidate") candidates.append({"seed": seed, **best, "path": str((seed_dir/best["path"]).relative_to(args.output))}) check_source() scope = ("diagnostic_pilot" if cfg["format"] == "skillfactory.contact-cycle-diagnostic-training.v1" else "three_seed_study") locked = {"candidates": candidates, "training_scope": scope, "accepted_skill": False, "independent_test_executed": False, "validation_screen_passed": all(c["failure_fraction"] <= 0.05 for c in candidates), "rule": "Never reselect these candidates using independent test outcomes"} path = args.output/"locked_candidates.json" if path.exists() and json.loads(path.read_text()) != locked: raise RuntimeError("Cannot overwrite a previous candidate lock") ledger.write_json(path, locked) return {"passed": True, "training_complete": True, "training_scope": scope, "accepted_skill": False, "locked": locked, "budget": ledger.data} def main(): parser = argparse.ArgumentParser(description=__doc__) mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--prepare", action="store_true") mode.add_argument("--train", action="store_true") parser.add_argument("--config", type=Path, default=ROOT/"configs/contact_cycle/training_v1.json") parser.add_argument("--output", type=Path, required=True) parser.add_argument("--authorization", type=Path) parser.add_argument("--resume", action="store_true") parser.add_argument("--console-log", type=Path) parser.add_argument("--headless", action="store_true") parser.add_argument("--device", default="cuda:0") args = parser.parse_args() frozen = (ROOT/"releases").resolve() if args.output.resolve() == frozen or frozen in args.output.resolve().parents: parser.error("Preparation/training must not write into frozen releases") from SkillFactory.legs.contact_cycle.training import ( BudgetLedger, digest, make_manifest, require_authorization, ) manifest = make_manifest(ROOT, args.config) cfg = manifest["training"] if args.prepare: if args.resume or args.authorization is not None: parser.error("Preparation neither resumes nor authorizes training") snapshot_size = sum((ROOT/p).stat().st_size for p in manifest["sources"]) if snapshot_size + len(json.dumps(manifest).encode()) + 1024*1024 > cfg["max_artifact_bytes"]: raise ValueError("Preparation snapshots exceed total artifact budget") args.output.mkdir(parents=True, exist_ok=False) write_json(args.output/"manifest.json", manifest) for relative in manifest["sources"]: target = args.output/"source_snapshot"/relative target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(ROOT/relative, target) write_json(args.output/"preparation.json", {"prepared": True, "authorized": False, "trained": False, "manifest_sha256": digest(manifest)}) print(f"Prepared only; no Kit/optimizer/independent test: {args.output}", flush=True) return # Fail BEFORE importing/launching Isaac/Kit and BEFORE starting the budget clock. grant = require_authorization(args.authorization, manifest) if not (args.output/"manifest.json").is_file() or json.loads((args.output/"manifest.json").read_text()) != manifest: raise ValueError("Use an immutable prepared directory matching the current sources/config") if (args.output/"budget.json").exists() != args.resume: raise ValueError("Existing experiment requires --resume; new experiment must not use it") if not args.headless: raise ValueError("Initial authorized run is headless only") if args.console_log is None or not args.console_log.resolve().is_relative_to(args.output.resolve()): raise ValueError("Supervised console log must be recorded inside the shared artifact budget directory") app = env = safety = None result = {"passed": False, "training_complete": False, "accepted_skill": False, "training_scope": ("diagnostic_pilot" if cfg["format"] == "skillfactory.contact-cycle-diagnostic-training.v1" else "three_seed_study")} code = 1 with BudgetLedger(args.output, manifest) as ledger: authorization_record = args.output/"authorization_record.json" if authorization_record.exists() and json.loads(authorization_record.read_text()) != grant: raise ValueError("Cannot replace the experiment's authorization record") ledger.write_json(authorization_record, grant) previous_result = args.output/"result.json" if previous_result.exists(): history = args.output/"attempt_history" history.mkdir(exist_ok=True) number = len(list(history.glob("result_*.json"))) + 1 ledger.write_json(history/f"result_{number:04d}.json", json.loads(previous_result.read_text())) try: from isaaclab.app import AppLauncher app = AppLauncher(headless=True, device=args.device).app from SkillFactory.tasks.manager_based.contact_cycle.cfg import ContactCycleEnvCfg from SkillFactory.tasks.manager_based.contact_cycle.env import ContactCycleEnv environment_cfg = ContactCycleEnvCfg(cycle_config=str(ROOT/cfg["task_config"])) environment_cfg.scene.num_envs = cfg["num_envs"] environment_cfg.sim.device = args.device environment_cfg.seed = cfg["seeds"][0] env = ContactCycleEnv(cfg=environment_cfg) from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit thresholds = json.loads((ROOT/cfg["proposal"]).read_text())["gates"] with PhysicalSafetyAudit(env, thresholds) as safety: result = execute(env, cfg, args, manifest, ledger, safety) code = 0 except BaseException: result["error"] = traceback.format_exc() traceback.print_exc() finally: result["budget"] = ledger.data if safety is not None: result["physics_safety"] = safety.report() if app is not None: finish_run(app, env, args.output/"result.json", result, code) else: write_json(args.output/"result.json", result) if code: sys.exit(code) if __name__ == "__main__": main()