Files
SkillFactory/scripts/train_contact_cycle.py
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

304 lines
16 KiB
Python

#!/usr/bin/env python3
"""Explicitly authorized contact-cycle PPO. --prepare does not launch Kit or train.
Independent testing is intentionally not an option. A continuation request or a
previous standing authorization is NOT a grant for this runner.
"""
import argparse
import hashlib
import json
import math
import shutil
import signal
import sys
import traceback
from pathlib import Path
from runtime_common import critical_log_lines, finish_run, write_json
ROOT = Path(__file__).resolve().parents[1]
def execute(env, cfg, args, manifest, ledger, safety):
import torch
from SkillFactory.legs.contact_cycle.curriculum import EpisodeSampler
from SkillFactory.legs.contact_cycle.ppo import (
CycleActorCritic,
collect_rollout,
features,
ppo_step,
restore_checkpoint,
restore_rng,
rng_state,
rollout_diagnostics,
save_checkpoint,
selection_score,
)
from SkillFactory.legs.contact_cycle.training import artifact_bytes, save_trace_subset, source_identity
from SkillFactory.tasks.manager_based.contact_cycle.mdp import state
from SkillFactory.tasks.manager_based.contact_cycle.recording import CycleRecorder
torch.set_num_threads(4)
s = state(env)
if (s.robot.is_fixed_base or env.cfg.sim.physics.solver_type != 0
or env.cfg.sim.physics.enable_external_forces_every_iteration):
raise RuntimeError("Require validated floating-root PGS/external-force physics settings")
stop = {"requested": False}
signal.signal(signal.SIGTERM, lambda *_: stop.update(requested=True))
signal.signal(signal.SIGINT, lambda *_: stop.update(requested=True))
runtime = {"torch": str(torch.__version__), "device": str(env.device), "physics_dt": env.physics_dt,
"control_dt": env.step_dt, "num_envs": env.num_envs}
identity = {"manifest": manifest, "runtime": runtime}
runtime_path = args.output/"runtime_identity.json"
if runtime_path.exists() and json.loads(runtime_path.read_text()) != runtime:
raise RuntimeError("Runtime identity changed across resume")
ledger.write_json(runtime_path, runtime)
thresholds = json.loads((ROOT/cfg["proposal"]).read_text())["gates"]
algo = {**cfg["algorithm"], "horizon": cfg["horizon"]}
def guard():
ledger.guard()
safety.check()
if stop["requested"]:
raise InterruptedError("Operator requested stop; resume from last atomic learner checkpoint")
def check_source():
guard()
if source_identity(ROOT, cfg, args.config) != manifest["sources"]:
raise RuntimeError("Training inputs changed; refusing continuation/checkpoint")
if args.console_log is not None and critical_log_lines(args.console_log):
raise RuntimeError("Critical runtime log; refusing successful completion")
def save(path, model, optimizer, seed_identity, update, sampler, best):
guard()
remaining = cfg["max_artifact_bytes"] - artifact_bytes(args.output) - 1024*1024
save_checkpoint(path, model, optimizer, seed_identity, update, sampler, best, ledger.data,
max_bytes=remaining)
ledger.guard()
candidates = []
for seed in cfg["seeds"]:
guard()
seed_dir = args.output/f"seed_{seed}"
seed_dir.mkdir(exist_ok=True)
seed_identity = {**identity, "seed": seed}
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
model = CycleActorCritic(cfg["algorithm"]["hidden"], cfg["algorithm"]["initial_std"]).to(env.device)
optimizer = torch.optim.Adam(model.parameters(), lr=cfg["algorithm"]["learning_rate"], eps=1e-5)
sampler = EpisodeSampler(s.cc, env.num_envs, seed)
latest = seed_dir/"latest.pt"
update, best = 0, None
if latest.exists():
checkpoint = restore_checkpoint(latest, model, optimizer, seed_identity, sampler)
update, best = checkpoint["update"], checkpoint["best"]
if ledger.data["pending"] is not None and ledger.data["pending"]["seed"] != seed:
if update != cfg["updates_per_seed"]:
raise RuntimeError("Unexpected unfinished earlier seed")
else:
ledger.reconcile_checkpoint(seed, update)
s.set_sampler(sampler)
x = features(env.reset()[0]["policy"])
if not latest.exists():
save(latest, model, optimizer, seed_identity, 0, sampler, None)
def log(row):
ledger.append_json(seed_dir/"metrics.jsonl", row)
@torch.no_grad()
def validation():
check_source()
saved_rng = rng_state()
validation_sampler = EpisodeSampler(s.cc, env.num_envs, cfg["validation_seed"], split="validation")
s.set_sampler(validation_sampler)
observation = env.reset()[0]["policy"]
inputs = {"scenario": s.scenarios.cpu().tolist(), "commands": s.schedule.commands.cpu().tolist(),
"points": s.sampled_points.cpu().tolist(), "delays": s.schedule.delays.cpu().tolist()}
if set(inputs["scenario"]) != set(range(4)):
raise RuntimeError("Validation family coverage missing")
steps = round(sum(s.cc["phase_durations_s"])/env.step_dt)
try:
with CycleRecorder(env) as recorder:
for step in range(steps):
guard()
action, mean, _, _ = model.act(features(observation), deterministic=True)
recorder.before_control(observation, action, mean,
model.log_std.clamp(math.log(0.02), math.log(0.5)).exp())
obs, _, terminated, truncated, _ = env.step(action)
recorder.end_control(terminated, truncated, step == steps - 1)
observation = obs["policy"]
rows, trace = recorder.summarize(thresholds)
score = selection_score(rows)
row = {"kind": "validation_only", "seed": seed, "validation_seed": cfg["validation_seed"],
"update": update, "reservation": ledger.data["reservations"], "score": score,
"failure_fraction": sum(not r["success"] for r in rows)/len(rows),
"inputs": inputs, "rows": rows, "accepted_skill": False}
label = f"validation_u{update:04d}_r{ledger.data['reservations']:06d}"
ledger.write_json(seed_dir/f"{label}.json", row)
# Keep bounded real failure traces: first four failed first episodes (or first four successes).
selected = ([r["env"] for r in rows if not r["success"]] or list(range(env.num_envs)))[:4]
save_trace_subset(seed_dir/f"{label}_trace.pt", trace, selected, recorder.stops,
lambda size: ledger.guard(reserve_bytes=size + 1024*1024))
save_trace_subset(seed_dir/f"{label}_initial.pt",
{k: v[None] for k, v in recorder.initial_state.items()}, selected,
[1]*env.num_envs, lambda size: ledger.guard(reserve_bytes=size + 1024*1024))
ledger.guard()
return row
finally:
s.set_sampler(sampler)
restore_rng(saved_rng)
while update < cfg["updates_per_seed"]:
check_source()
sampler.set_update(update + 1)
ledger.reserve_rollout(seed, update + 1)
data, x, failures, timeouts = collect_rollout(env, model, x, algo, guard)
guard()
diagnostic_row = rollout_diagnostics(data, algo)
if diagnostic_row["missing_diagnostic_samples"]:
raise RuntimeError("Missing control-clock diagnostics; refusing optimizer update")
learning = ppo_step(model, optimizer, data, algo)
update += 1
if update % cfg["validation_interval"] == 0:
validation_row = validation()
log(validation_row)
if best is None or validation_row["score"] < best["score"]:
relative = f"candidate_u{update:04d}_r{ledger.data['reservations']:06d}.pt"
best = {"update": update, "score": validation_row["score"], "path": relative,
"failure_fraction": validation_row["failure_fraction"]}
save(seed_dir/relative, model, optimizer, seed_identity, update, sampler, best)
best["sha256"] = hashlib.sha256((seed_dir/relative).read_bytes()).hexdigest()
x = features(env.reset()[0]["policy"])
# Checkpoint BEFORE durable progress commit; ledger reconciles the crash window on resume.
check_source()
save(latest, model, optimizer, seed_identity, update, sampler, best)
ledger.commit_update(seed, update)
log({"kind": "training", "update": update, "seed": seed, "learner": learning,
"terminated": failures, "timeouts": timeouts, "curriculum_stage": sampler.stage,
"episode_counts": sampler.episodes.tolist(),
"diagnostics": diagnostic_row,
"reserved_transitions": ledger.data["reserved_transitions"]})
if best is None or hashlib.sha256((seed_dir/best["path"]).read_bytes()).hexdigest() != best["sha256"]:
raise RuntimeError("Missing/changed fixed candidate")
candidates.append({"seed": seed, **best, "path": str((seed_dir/best["path"]).relative_to(args.output))})
check_source()
scope = ("diagnostic_pilot" if cfg["format"] == "skillfactory.contact-cycle-diagnostic-training.v1"
else "three_seed_study")
locked = {"candidates": candidates, "training_scope": scope,
"accepted_skill": False, "independent_test_executed": False,
"validation_screen_passed": all(c["failure_fraction"] <= 0.05 for c in candidates),
"rule": "Never reselect these candidates using independent test outcomes"}
path = args.output/"locked_candidates.json"
if path.exists() and json.loads(path.read_text()) != locked:
raise RuntimeError("Cannot overwrite a previous candidate lock")
ledger.write_json(path, locked)
return {"passed": True, "training_complete": True, "training_scope": scope,
"accepted_skill": False, "locked": locked,
"budget": ledger.data}
def main():
parser = argparse.ArgumentParser(description=__doc__)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--prepare", action="store_true")
mode.add_argument("--train", action="store_true")
parser.add_argument("--config", type=Path, default=ROOT/"configs/contact_cycle/training_v1.json")
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--authorization", type=Path)
parser.add_argument("--resume", action="store_true")
parser.add_argument("--console-log", type=Path)
parser.add_argument("--headless", action="store_true")
parser.add_argument("--device", default="cuda:0")
args = parser.parse_args()
frozen = (ROOT/"releases").resolve()
if args.output.resolve() == frozen or frozen in args.output.resolve().parents:
parser.error("Preparation/training must not write into frozen releases")
from SkillFactory.legs.contact_cycle.training import (
BudgetLedger,
digest,
make_manifest,
require_authorization,
)
manifest = make_manifest(ROOT, args.config)
cfg = manifest["training"]
if args.prepare:
if args.resume or args.authorization is not None:
parser.error("Preparation neither resumes nor authorizes training")
snapshot_size = sum((ROOT/p).stat().st_size for p in manifest["sources"])
if snapshot_size + len(json.dumps(manifest).encode()) + 1024*1024 > cfg["max_artifact_bytes"]:
raise ValueError("Preparation snapshots exceed total artifact budget")
args.output.mkdir(parents=True, exist_ok=False)
write_json(args.output/"manifest.json", manifest)
for relative in manifest["sources"]:
target = args.output/"source_snapshot"/relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(ROOT/relative, target)
write_json(args.output/"preparation.json", {"prepared": True, "authorized": False, "trained": False,
"manifest_sha256": digest(manifest)})
print(f"Prepared only; no Kit/optimizer/independent test: {args.output}", flush=True)
return
# Fail BEFORE importing/launching Isaac/Kit and BEFORE starting the budget clock.
grant = require_authorization(args.authorization, manifest)
if not (args.output/"manifest.json").is_file() or json.loads((args.output/"manifest.json").read_text()) != manifest:
raise ValueError("Use an immutable prepared directory matching the current sources/config")
if (args.output/"budget.json").exists() != args.resume:
raise ValueError("Existing experiment requires --resume; new experiment must not use it")
if not args.headless:
raise ValueError("Initial authorized run is headless only")
if args.console_log is None or not args.console_log.resolve().is_relative_to(args.output.resolve()):
raise ValueError("Supervised console log must be recorded inside the shared artifact budget directory")
app = env = safety = None
result = {"passed": False, "training_complete": False, "accepted_skill": False,
"training_scope": ("diagnostic_pilot"
if cfg["format"] == "skillfactory.contact-cycle-diagnostic-training.v1"
else "three_seed_study")}
code = 1
with BudgetLedger(args.output, manifest) as ledger:
authorization_record = args.output/"authorization_record.json"
if authorization_record.exists() and json.loads(authorization_record.read_text()) != grant:
raise ValueError("Cannot replace the experiment's authorization record")
ledger.write_json(authorization_record, grant)
previous_result = args.output/"result.json"
if previous_result.exists():
history = args.output/"attempt_history"
history.mkdir(exist_ok=True)
number = len(list(history.glob("result_*.json"))) + 1
ledger.write_json(history/f"result_{number:04d}.json", json.loads(previous_result.read_text()))
try:
from isaaclab.app import AppLauncher
app = AppLauncher(headless=True, device=args.device).app
from SkillFactory.tasks.manager_based.contact_cycle.cfg import ContactCycleEnvCfg
from SkillFactory.tasks.manager_based.contact_cycle.env import ContactCycleEnv
environment_cfg = ContactCycleEnvCfg(cycle_config=str(ROOT/cfg["task_config"]))
environment_cfg.scene.num_envs = cfg["num_envs"]
environment_cfg.sim.device = args.device
environment_cfg.seed = cfg["seeds"][0]
env = ContactCycleEnv(cfg=environment_cfg)
from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit
thresholds = json.loads((ROOT/cfg["proposal"]).read_text())["gates"]
with PhysicalSafetyAudit(env, thresholds) as safety:
result = execute(env, cfg, args, manifest, ledger, safety)
code = 0
except BaseException:
result["error"] = traceback.format_exc()
traceback.print_exc()
finally:
result["budget"] = ledger.data
if safety is not None:
result["physics_safety"] = safety.report()
if app is not None:
finish_run(app, env, args.output/"result.json", result, code)
else:
write_json(args.output/"result.json", result)
if code:
sys.exit(code)
if __name__ == "__main__":
main()