Files
SkillFactory/scripts/train_cycle_sim_residual.py
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

292 lines
15 KiB
Python

#!/usr/bin/env python3
"""Fresh 46D residual PPO pilot. --prepare neither launches Kit nor authorizes learning.
Only a parent-bound final manifest grant permits --train. No resume/warm-start CLI.
The sole actual training launch must be supervised with a <=3480s outer timeout.
"""
import argparse
import hashlib
import json
import math
import signal
import sys
import traceback
from pathlib import Path
from runtime_common import critical_log_lines
ROOT = Path(__file__).resolve().parents[1]
def execute(env, cfg, args, manifest, ledger, guard):
import torch
from audit_contact_cycle_functional import evaluate
from SkillFactory.legs.contact_cycle_sim.model import SimResidualActorCritic
from SkillFactory.legs.contact_cycle_sim.ppo import (
features,
save_checkpoint,
selection_key,
)
from SkillFactory.legs.contact_cycle_sim.training import (
MODEL_CONFIG,
artifact_bytes,
checkpoint_then_commit,
choose_candidate,
isolated_validation_rng,
optimize_reserved_rollout,
parameter_deltas,
save_trace_subset,
source_identity,
validation_rows,
verify_snapshot,
)
from SkillFactory.tasks.manager_based.contact_cycle_sim.mdp import state
from SkillFactory.tasks.manager_based.contact_cycle_sim.recording import SimCycleRecorder
torch.set_num_threads(4)
s = state(env)
if (s.robot.is_fixed_base or env.cfg.sim.physics.solver_type != 0
or env.cfg.sim.physics.enable_external_forces_every_iteration
or env.cfg.terminations.failed is None or env.physics_dt != .002 or env.step_dt != .01
or env.num_envs != 128 or env.cfg.episode_length_s != 9):
raise RuntimeError("Require unchanged floating-root PGS, active failures, 500Hz physics and 9s nominal task")
runtime = {"torch": str(torch.__version__), "device": str(env.device), "physics_dt": env.physics_dt,
"control_dt": env.step_dt, "num_envs": env.num_envs}
identity = {"manifest": manifest, "runtime": runtime, "seed": cfg["seeds"][0], "model_config": MODEL_CONFIG}
ledger.write_json(args.output / "runtime_identity.json", identity)
functional = json.loads((ROOT / cfg["functional_config"]).read_text())
algo = {**cfg["algorithm"], "horizon": cfg["horizon"]}
seed = cfg["seeds"][0]
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
model = SimResidualActorCritic(**MODEL_CONFIG).to(env.device)
initial = {k: v.detach().cpu().clone() for k, v in model.named_parameters()}
optimizer = torch.optim.Adam(model.parameters(), lr=algo["learning_rate"], eps=algo["adam_eps"])
best = None
def check_source():
guard()
if source_identity(ROOT, cfg, args.config) != manifest["sources"]:
raise RuntimeError("Bound sources/assets/config changed; refusing optimization or publication")
verify_snapshot(args.output, manifest)
if critical_log_lines(args.console_log):
raise RuntimeError("Critical simulator console log")
def save(path, update):
check_source()
remaining = cfg["max_artifact_bytes"] - artifact_bytes(args.output) - 1048576
size = save_checkpoint(path, model, optimizer, identity, update, best, ledger.data,
MODEL_CONFIG, max_bytes=remaining)
ledger.guard()
return size
@torch.no_grad()
def validation(update):
check_source()
with isolated_validation_rng(env, cfg["validation_seed"], seed):
observation = features(env.reset()[0]["policy"])
if bool(s.scenarios.any() or s.stand_only.any()):
raise RuntimeError("Validation must contain only nominal complete cycles")
returns = torch.zeros(env.num_envs, device=env.device)
live = torch.ones(env.num_envs, dtype=torch.bool, device=env.device)
with SimCycleRecorder(env) as recorder:
for step in range(900):
guard()
action, mean, _, _ = model.act(observation, deterministic=True)
recorder.before_control(observation, action, mean,
model.log_std.clamp(math.log(.02), math.log(.5)).exp())
obs, reward, terminated, truncated, _ = env.step(action)
if not torch.isfinite(reward).all():
raise RuntimeError("Nonfinite validation reward")
returns += reward * live
live &= ~(terminated | truncated)
recorder.end_control(terminated, truncated, step == 899)
observation = features(obs["policy"])
trace = {k: torch.stack([frame[k] for frame in recorder.frames]) for k in recorder.frames[0]}
# Recorded buffers are copies; release the original per-frame storage before serialization.
recorder.frames.clear()
rows = validation_rows(trace, recorder.stops, recorder.early, s.nominal, functional, evaluate)
for row, total in zip(rows, returns.cpu().tolist()):
row["episode_return"] = total
result = {"kind": ("untrained_zero_residual_network" if update == 0
else "analytic_reference_plus_learned_residual"),
"trained": update > 0, "update": update, "seed": seed,
"validation_seed": cfg["validation_seed"], "rows": rows,
"selection_key": list(selection_key(rows)), "parameter_deltas": parameter_deltas(model, initial),
"accepted_skill": False, "independent_test_executed": False,
"audit_controller_kind": "learned_policy_metadata_interface_only"}
label = f"validation_u{update:04d}"
ledger.write_json(args.output / f"{label}.json", result)
selected = ([r["env"] for r in rows if not r["success"]] or list(range(env.num_envs)))[:4]
def reserve(size):
ledger.guard(reserve_bytes=size + 1048576)
save_trace_subset(args.output / f"{label}_trace.pt", trace, selected, recorder.stops, reserve)
save_trace_subset(args.output / f"{label}_initial.pt",
{k: v[None] for k, v in recorder.initial_state.items()}, selected,
[1] * env.num_envs, reserve)
check_source()
return result
save(args.output / "untrained_u0000.pt", 0)
baseline = validation(0)
if not all(r["success"] for r in baseline["rows"]) or len(baseline["rows"]) != 128:
raise RuntimeError("u0 did not pass 128/128 functional cycles; stop before the first training rollout")
comparisons = []
x = features(env.reset()[0]["policy"])
for update in range(1, cfg["updates_per_seed"] + 1):
x, failures, timeouts, learning = optimize_reserved_rollout(
env, model, optimizer, x, algo, seed, update, ledger, guard, check_source)
if update in cfg["validation_updates"]:
row = validation(update)
if choose_candidate(best, update, row["rows"]):
name = f"candidate_u{update:04d}.pt"
best = {"update": update, "selection_key": row["selection_key"], "path": name}
save(args.output / name, update)
best["sha256"] = hashlib.sha256((args.output / name).read_bytes()).hexdigest()
comparisons.append({"update": update, "selection_key": row["selection_key"],
"parameter_deltas": row["parameter_deltas"],
"mean_height_rms_m": sum(r["height_rms_m"] for r in row["rows"]) / 128,
"mean_return": sum(r["episode_return"] for r in row["rows"]) / 128,
"mean_actual_residual_rms_rad": sum(
r["actual_residual_rms_rad"] for r in row["rows"]) / 128})
x = features(env.reset()[0]["policy"])
# Pending reservation remains charged on ANY failure; no automatic recovery.
latest_bytes = checkpoint_then_commit(
ledger, seed, update, lambda: save(args.output / "latest.pt", update))
ledger.append_json(args.output / "metrics.jsonl", {"update": update, "learner": learning,
"failures": failures, "timeouts": timeouts, "latest_checkpoint_bytes": latest_bytes,
"reserved_transitions": ledger.data["reserved_transitions"]})
check_source()
if (best is None or best["update"] == 0
or hashlib.sha256((args.output / best["path"]).read_bytes()).hexdigest() != best["sha256"]):
raise RuntimeError("Missing or changed trained candidate")
locked = {"candidate": best, "accepted_skill": False, "independent_test_executed": False,
"training_scope": "nominal_simulation_residual_pilot",
"rule": "failure fraction then all-row cost; earliest tie; never reselect from later outcomes"}
ledger.write_json(args.output / "locked_candidate.json", locked)
ledger.write_json(args.output / "baseline_comparison.json", {
"baseline_kind": baseline["kind"], "baseline_selection_key": baseline["selection_key"],
"baseline_mean_height_rms_m": sum(r["height_rms_m"] for r in baseline["rows"]) / 128,
"baseline_mean_return": sum(r["episode_return"] for r in baseline["rows"]) / 128,
"baseline_mean_actual_residual_rms_rad": sum(r["actual_residual_rms_rad"] for r in baseline["rows"]) / 128,
"updated_models": comparisons, "selection_changed_by_comparison": False,
"note": "Reference already passed at u0. Nonzero parameter deltas alone do not demonstrate improvement.",
"accepted_skill": False})
return {"passed": True, "training_complete": True, "accepted_skill": False, "locked": locked}
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--prepare", action="store_true")
mode.add_argument("--train", action="store_true")
parser.add_argument("--config", type=Path, default=ROOT / "configs/contact_cycle/sim_training_v1.json")
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--authorization", type=Path)
parser.add_argument("--console-log", type=Path)
parser.add_argument("--supervisor-report", type=Path)
parser.add_argument("--headless", action="store_true")
parser.add_argument("--device", default="cuda:0")
args = parser.parse_args(argv)
from SkillFactory.legs.contact_cycle_sim.training import (
FreshBudgetLedger,
assert_no_conflicting_processes,
make_manifest,
prepare,
require_authorization,
validate_prepared,
writable_output,
)
args.output = writable_output(ROOT, args.output)
if args.prepare:
if args.authorization is not None or args.console_log is not None or args.supervisor_report is not None:
parser.error("Preparation is not authorization or a supervised launch")
prepare(ROOT, args.config, args.output)
print(f"Prepared only; no Kit, budget or optimizer: {args.output}", flush=True)
return
# Every gate here runs BEFORE importing Kit and BEFORE the ledger clock starts.
manifest = make_manifest(ROOT, args.config)
grant = require_authorization(args.authorization, manifest)
validate_prepared(ROOT, args.output, manifest)
if not args.headless:
raise ValueError("Pilot must be headless and externally supervised")
if (args.console_log is None or args.console_log.resolve() != args.output / "console.log"
or args.supervisor_report is None or args.supervisor_report.resolve() != args.output / "supervisor.json"
or not args.console_log.is_file() or not args.supervisor_report.is_file()):
raise ValueError("Require live supervisor console.log and supervisor.json inside artifact budget")
supervisor = json.loads(args.supervisor_report.read_text())
if supervisor.get("status") != "running" or supervisor.get("passed") is not False:
raise ValueError("Require a running external supervisor")
assert_no_conflicting_processes()
cfg = manifest["training"]
stop = {"requested": False}
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: stop.update(requested=True))
app = env = safety = None
result = {"passed": False, "training_complete": False, "accepted_skill": False}
code = 1
with FreshBudgetLedger(args.output, manifest) as ledger:
def guard():
ledger.guard()
if stop["requested"]:
raise InterruptedError("Cooperative stop; pending work remains charged; no retry/resume")
if safety is not None:
safety.check()
ledger.write_json(args.output / "authorization_record.json", grant)
try:
guard()
from isaaclab.app import AppLauncher
app = AppLauncher(headless=True, device=args.device).app
guard()
from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit
from SkillFactory.tasks.manager_based.contact_cycle_sim.cfg import SimContactCycleEnvCfg
from SkillFactory.tasks.manager_based.contact_cycle_sim.env import SimContactCycleEnv
environment_cfg = SimContactCycleEnvCfg(residual_config=str(ROOT / cfg["task_config"]))
environment_cfg.scene.num_envs = cfg["num_envs"]
environment_cfg.sim.device = args.device
environment_cfg.seed = cfg["seeds"][0]
env = SimContactCycleEnv(cfg=environment_cfg)
thresholds = json.loads((ROOT / cfg["safety_config"]).read_text())["gates"]
with PhysicalSafetyAudit(env, thresholds) as safety:
result = execute(env, cfg, args, manifest, ledger, guard)
guard()
code = 0
except BaseException:
result.update(passed=False, error=traceback.format_exc())
traceback.print_exc()
finally:
if env is not None:
try:
env.close()
except BaseException:
result.update(passed=False, shutdown_error=traceback.format_exc())
code = 1
result["budget"] = ledger.data
if safety is not None:
result["physics_safety"] = safety.report()
try:
guard()
if critical_log_lines(args.console_log):
raise RuntimeError("Critical console log during shutdown")
except BaseException:
result.update(passed=False, final_guard_error=traceback.format_exc())
code = 1
result["status"] = "completed" if code == 0 else "failed"
# Never bypass the artifact/deadline guard to publish a success on failure.
try:
ledger.write_json(args.output / "result.json", result)
finally:
if app is not None:
app.close(exit_code=code)
if code:
sys.exit(code)
if __name__ == "__main__":
main()