9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
304 lines
16 KiB
Python
304 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Explicitly authorized contact-cycle PPO. --prepare does not launch Kit or train.
|
|
|
|
Independent testing is intentionally not an option. A continuation request or a
|
|
previous standing authorization is NOT a grant for this runner.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import shutil
|
|
import signal
|
|
import sys
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
from runtime_common import critical_log_lines, finish_run, write_json
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def execute(env, cfg, args, manifest, ledger, safety):
|
|
import torch
|
|
from SkillFactory.legs.contact_cycle.curriculum import EpisodeSampler
|
|
from SkillFactory.legs.contact_cycle.ppo import (
|
|
CycleActorCritic,
|
|
collect_rollout,
|
|
features,
|
|
ppo_step,
|
|
restore_checkpoint,
|
|
restore_rng,
|
|
rng_state,
|
|
rollout_diagnostics,
|
|
save_checkpoint,
|
|
selection_score,
|
|
)
|
|
from SkillFactory.legs.contact_cycle.training import artifact_bytes, save_trace_subset, source_identity
|
|
from SkillFactory.tasks.manager_based.contact_cycle.mdp import state
|
|
from SkillFactory.tasks.manager_based.contact_cycle.recording import CycleRecorder
|
|
|
|
torch.set_num_threads(4)
|
|
s = state(env)
|
|
if (s.robot.is_fixed_base or env.cfg.sim.physics.solver_type != 0
|
|
or env.cfg.sim.physics.enable_external_forces_every_iteration):
|
|
raise RuntimeError("Require validated floating-root PGS/external-force physics settings")
|
|
stop = {"requested": False}
|
|
signal.signal(signal.SIGTERM, lambda *_: stop.update(requested=True))
|
|
signal.signal(signal.SIGINT, lambda *_: stop.update(requested=True))
|
|
runtime = {"torch": str(torch.__version__), "device": str(env.device), "physics_dt": env.physics_dt,
|
|
"control_dt": env.step_dt, "num_envs": env.num_envs}
|
|
identity = {"manifest": manifest, "runtime": runtime}
|
|
runtime_path = args.output/"runtime_identity.json"
|
|
if runtime_path.exists() and json.loads(runtime_path.read_text()) != runtime:
|
|
raise RuntimeError("Runtime identity changed across resume")
|
|
ledger.write_json(runtime_path, runtime)
|
|
thresholds = json.loads((ROOT/cfg["proposal"]).read_text())["gates"]
|
|
algo = {**cfg["algorithm"], "horizon": cfg["horizon"]}
|
|
|
|
def guard():
|
|
ledger.guard()
|
|
safety.check()
|
|
if stop["requested"]:
|
|
raise InterruptedError("Operator requested stop; resume from last atomic learner checkpoint")
|
|
|
|
def check_source():
|
|
guard()
|
|
if source_identity(ROOT, cfg, args.config) != manifest["sources"]:
|
|
raise RuntimeError("Training inputs changed; refusing continuation/checkpoint")
|
|
if args.console_log is not None and critical_log_lines(args.console_log):
|
|
raise RuntimeError("Critical runtime log; refusing successful completion")
|
|
|
|
def save(path, model, optimizer, seed_identity, update, sampler, best):
|
|
guard()
|
|
remaining = cfg["max_artifact_bytes"] - artifact_bytes(args.output) - 1024*1024
|
|
save_checkpoint(path, model, optimizer, seed_identity, update, sampler, best, ledger.data,
|
|
max_bytes=remaining)
|
|
ledger.guard()
|
|
|
|
candidates = []
|
|
for seed in cfg["seeds"]:
|
|
guard()
|
|
seed_dir = args.output/f"seed_{seed}"
|
|
seed_dir.mkdir(exist_ok=True)
|
|
seed_identity = {**identity, "seed": seed}
|
|
torch.manual_seed(seed)
|
|
torch.cuda.manual_seed_all(seed)
|
|
model = CycleActorCritic(cfg["algorithm"]["hidden"], cfg["algorithm"]["initial_std"]).to(env.device)
|
|
optimizer = torch.optim.Adam(model.parameters(), lr=cfg["algorithm"]["learning_rate"], eps=1e-5)
|
|
sampler = EpisodeSampler(s.cc, env.num_envs, seed)
|
|
latest = seed_dir/"latest.pt"
|
|
update, best = 0, None
|
|
if latest.exists():
|
|
checkpoint = restore_checkpoint(latest, model, optimizer, seed_identity, sampler)
|
|
update, best = checkpoint["update"], checkpoint["best"]
|
|
if ledger.data["pending"] is not None and ledger.data["pending"]["seed"] != seed:
|
|
if update != cfg["updates_per_seed"]:
|
|
raise RuntimeError("Unexpected unfinished earlier seed")
|
|
else:
|
|
ledger.reconcile_checkpoint(seed, update)
|
|
s.set_sampler(sampler)
|
|
x = features(env.reset()[0]["policy"])
|
|
if not latest.exists():
|
|
save(latest, model, optimizer, seed_identity, 0, sampler, None)
|
|
|
|
def log(row):
|
|
ledger.append_json(seed_dir/"metrics.jsonl", row)
|
|
|
|
@torch.no_grad()
|
|
def validation():
|
|
check_source()
|
|
saved_rng = rng_state()
|
|
validation_sampler = EpisodeSampler(s.cc, env.num_envs, cfg["validation_seed"], split="validation")
|
|
s.set_sampler(validation_sampler)
|
|
observation = env.reset()[0]["policy"]
|
|
inputs = {"scenario": s.scenarios.cpu().tolist(), "commands": s.schedule.commands.cpu().tolist(),
|
|
"points": s.sampled_points.cpu().tolist(), "delays": s.schedule.delays.cpu().tolist()}
|
|
if set(inputs["scenario"]) != set(range(4)):
|
|
raise RuntimeError("Validation family coverage missing")
|
|
steps = round(sum(s.cc["phase_durations_s"])/env.step_dt)
|
|
try:
|
|
with CycleRecorder(env) as recorder:
|
|
for step in range(steps):
|
|
guard()
|
|
action, mean, _, _ = model.act(features(observation), deterministic=True)
|
|
recorder.before_control(observation, action, mean,
|
|
model.log_std.clamp(math.log(0.02), math.log(0.5)).exp())
|
|
obs, _, terminated, truncated, _ = env.step(action)
|
|
recorder.end_control(terminated, truncated, step == steps - 1)
|
|
observation = obs["policy"]
|
|
rows, trace = recorder.summarize(thresholds)
|
|
score = selection_score(rows)
|
|
row = {"kind": "validation_only", "seed": seed, "validation_seed": cfg["validation_seed"],
|
|
"update": update, "reservation": ledger.data["reservations"], "score": score,
|
|
"failure_fraction": sum(not r["success"] for r in rows)/len(rows),
|
|
"inputs": inputs, "rows": rows, "accepted_skill": False}
|
|
label = f"validation_u{update:04d}_r{ledger.data['reservations']:06d}"
|
|
ledger.write_json(seed_dir/f"{label}.json", row)
|
|
# Keep bounded real failure traces: first four failed first episodes (or first four successes).
|
|
selected = ([r["env"] for r in rows if not r["success"]] or list(range(env.num_envs)))[:4]
|
|
save_trace_subset(seed_dir/f"{label}_trace.pt", trace, selected, recorder.stops,
|
|
lambda size: ledger.guard(reserve_bytes=size + 1024*1024))
|
|
save_trace_subset(seed_dir/f"{label}_initial.pt",
|
|
{k: v[None] for k, v in recorder.initial_state.items()}, selected,
|
|
[1]*env.num_envs, lambda size: ledger.guard(reserve_bytes=size + 1024*1024))
|
|
ledger.guard()
|
|
return row
|
|
finally:
|
|
s.set_sampler(sampler)
|
|
restore_rng(saved_rng)
|
|
|
|
while update < cfg["updates_per_seed"]:
|
|
check_source()
|
|
sampler.set_update(update + 1)
|
|
ledger.reserve_rollout(seed, update + 1)
|
|
data, x, failures, timeouts = collect_rollout(env, model, x, algo, guard)
|
|
guard()
|
|
diagnostic_row = rollout_diagnostics(data, algo)
|
|
if diagnostic_row["missing_diagnostic_samples"]:
|
|
raise RuntimeError("Missing control-clock diagnostics; refusing optimizer update")
|
|
learning = ppo_step(model, optimizer, data, algo)
|
|
update += 1
|
|
if update % cfg["validation_interval"] == 0:
|
|
validation_row = validation()
|
|
log(validation_row)
|
|
if best is None or validation_row["score"] < best["score"]:
|
|
relative = f"candidate_u{update:04d}_r{ledger.data['reservations']:06d}.pt"
|
|
best = {"update": update, "score": validation_row["score"], "path": relative,
|
|
"failure_fraction": validation_row["failure_fraction"]}
|
|
save(seed_dir/relative, model, optimizer, seed_identity, update, sampler, best)
|
|
best["sha256"] = hashlib.sha256((seed_dir/relative).read_bytes()).hexdigest()
|
|
x = features(env.reset()[0]["policy"])
|
|
# Checkpoint BEFORE durable progress commit; ledger reconciles the crash window on resume.
|
|
check_source()
|
|
save(latest, model, optimizer, seed_identity, update, sampler, best)
|
|
ledger.commit_update(seed, update)
|
|
log({"kind": "training", "update": update, "seed": seed, "learner": learning,
|
|
"terminated": failures, "timeouts": timeouts, "curriculum_stage": sampler.stage,
|
|
"episode_counts": sampler.episodes.tolist(),
|
|
"diagnostics": diagnostic_row,
|
|
"reserved_transitions": ledger.data["reserved_transitions"]})
|
|
if best is None or hashlib.sha256((seed_dir/best["path"]).read_bytes()).hexdigest() != best["sha256"]:
|
|
raise RuntimeError("Missing/changed fixed candidate")
|
|
candidates.append({"seed": seed, **best, "path": str((seed_dir/best["path"]).relative_to(args.output))})
|
|
check_source()
|
|
scope = ("diagnostic_pilot" if cfg["format"] == "skillfactory.contact-cycle-diagnostic-training.v1"
|
|
else "three_seed_study")
|
|
locked = {"candidates": candidates, "training_scope": scope,
|
|
"accepted_skill": False, "independent_test_executed": False,
|
|
"validation_screen_passed": all(c["failure_fraction"] <= 0.05 for c in candidates),
|
|
"rule": "Never reselect these candidates using independent test outcomes"}
|
|
path = args.output/"locked_candidates.json"
|
|
if path.exists() and json.loads(path.read_text()) != locked:
|
|
raise RuntimeError("Cannot overwrite a previous candidate lock")
|
|
ledger.write_json(path, locked)
|
|
return {"passed": True, "training_complete": True, "training_scope": scope,
|
|
"accepted_skill": False, "locked": locked,
|
|
"budget": ledger.data}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
mode = parser.add_mutually_exclusive_group(required=True)
|
|
mode.add_argument("--prepare", action="store_true")
|
|
mode.add_argument("--train", action="store_true")
|
|
parser.add_argument("--config", type=Path, default=ROOT/"configs/contact_cycle/training_v1.json")
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--authorization", type=Path)
|
|
parser.add_argument("--resume", action="store_true")
|
|
parser.add_argument("--console-log", type=Path)
|
|
parser.add_argument("--headless", action="store_true")
|
|
parser.add_argument("--device", default="cuda:0")
|
|
args = parser.parse_args()
|
|
frozen = (ROOT/"releases").resolve()
|
|
if args.output.resolve() == frozen or frozen in args.output.resolve().parents:
|
|
parser.error("Preparation/training must not write into frozen releases")
|
|
from SkillFactory.legs.contact_cycle.training import (
|
|
BudgetLedger,
|
|
digest,
|
|
make_manifest,
|
|
require_authorization,
|
|
)
|
|
|
|
manifest = make_manifest(ROOT, args.config)
|
|
cfg = manifest["training"]
|
|
if args.prepare:
|
|
if args.resume or args.authorization is not None:
|
|
parser.error("Preparation neither resumes nor authorizes training")
|
|
snapshot_size = sum((ROOT/p).stat().st_size for p in manifest["sources"])
|
|
if snapshot_size + len(json.dumps(manifest).encode()) + 1024*1024 > cfg["max_artifact_bytes"]:
|
|
raise ValueError("Preparation snapshots exceed total artifact budget")
|
|
args.output.mkdir(parents=True, exist_ok=False)
|
|
write_json(args.output/"manifest.json", manifest)
|
|
for relative in manifest["sources"]:
|
|
target = args.output/"source_snapshot"/relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(ROOT/relative, target)
|
|
write_json(args.output/"preparation.json", {"prepared": True, "authorized": False, "trained": False,
|
|
"manifest_sha256": digest(manifest)})
|
|
print(f"Prepared only; no Kit/optimizer/independent test: {args.output}", flush=True)
|
|
return
|
|
# Fail BEFORE importing/launching Isaac/Kit and BEFORE starting the budget clock.
|
|
grant = require_authorization(args.authorization, manifest)
|
|
if not (args.output/"manifest.json").is_file() or json.loads((args.output/"manifest.json").read_text()) != manifest:
|
|
raise ValueError("Use an immutable prepared directory matching the current sources/config")
|
|
if (args.output/"budget.json").exists() != args.resume:
|
|
raise ValueError("Existing experiment requires --resume; new experiment must not use it")
|
|
if not args.headless:
|
|
raise ValueError("Initial authorized run is headless only")
|
|
if args.console_log is None or not args.console_log.resolve().is_relative_to(args.output.resolve()):
|
|
raise ValueError("Supervised console log must be recorded inside the shared artifact budget directory")
|
|
app = env = safety = None
|
|
result = {"passed": False, "training_complete": False, "accepted_skill": False,
|
|
"training_scope": ("diagnostic_pilot"
|
|
if cfg["format"] == "skillfactory.contact-cycle-diagnostic-training.v1"
|
|
else "three_seed_study")}
|
|
code = 1
|
|
with BudgetLedger(args.output, manifest) as ledger:
|
|
authorization_record = args.output/"authorization_record.json"
|
|
if authorization_record.exists() and json.loads(authorization_record.read_text()) != grant:
|
|
raise ValueError("Cannot replace the experiment's authorization record")
|
|
ledger.write_json(authorization_record, grant)
|
|
previous_result = args.output/"result.json"
|
|
if previous_result.exists():
|
|
history = args.output/"attempt_history"
|
|
history.mkdir(exist_ok=True)
|
|
number = len(list(history.glob("result_*.json"))) + 1
|
|
ledger.write_json(history/f"result_{number:04d}.json", json.loads(previous_result.read_text()))
|
|
try:
|
|
from isaaclab.app import AppLauncher
|
|
|
|
app = AppLauncher(headless=True, device=args.device).app
|
|
from SkillFactory.tasks.manager_based.contact_cycle.cfg import ContactCycleEnvCfg
|
|
from SkillFactory.tasks.manager_based.contact_cycle.env import ContactCycleEnv
|
|
|
|
environment_cfg = ContactCycleEnvCfg(cycle_config=str(ROOT/cfg["task_config"]))
|
|
environment_cfg.scene.num_envs = cfg["num_envs"]
|
|
environment_cfg.sim.device = args.device
|
|
environment_cfg.seed = cfg["seeds"][0]
|
|
env = ContactCycleEnv(cfg=environment_cfg)
|
|
from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit
|
|
|
|
thresholds = json.loads((ROOT/cfg["proposal"]).read_text())["gates"]
|
|
with PhysicalSafetyAudit(env, thresholds) as safety:
|
|
result = execute(env, cfg, args, manifest, ledger, safety)
|
|
code = 0
|
|
except BaseException:
|
|
result["error"] = traceback.format_exc()
|
|
traceback.print_exc()
|
|
finally:
|
|
result["budget"] = ledger.data
|
|
if safety is not None:
|
|
result["physics_safety"] = safety.report()
|
|
if app is not None:
|
|
finish_run(app, env, args.output/"result.json", result, code)
|
|
else:
|
|
write_json(args.output/"result.json", result)
|
|
if code:
|
|
sys.exit(code)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|