9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
292 lines
15 KiB
Python
292 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Fresh 46D residual PPO pilot. --prepare neither launches Kit nor authorizes learning.
|
|
|
|
Only a parent-bound final manifest grant permits --train. No resume/warm-start CLI.
|
|
The sole actual training launch must be supervised with a <=3480s outer timeout.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import signal
|
|
import sys
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
from runtime_common import critical_log_lines
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def execute(env, cfg, args, manifest, ledger, guard):
|
|
import torch
|
|
from audit_contact_cycle_functional import evaluate
|
|
from SkillFactory.legs.contact_cycle_sim.model import SimResidualActorCritic
|
|
from SkillFactory.legs.contact_cycle_sim.ppo import (
|
|
features,
|
|
save_checkpoint,
|
|
selection_key,
|
|
)
|
|
from SkillFactory.legs.contact_cycle_sim.training import (
|
|
MODEL_CONFIG,
|
|
artifact_bytes,
|
|
checkpoint_then_commit,
|
|
choose_candidate,
|
|
isolated_validation_rng,
|
|
optimize_reserved_rollout,
|
|
parameter_deltas,
|
|
save_trace_subset,
|
|
source_identity,
|
|
validation_rows,
|
|
verify_snapshot,
|
|
)
|
|
from SkillFactory.tasks.manager_based.contact_cycle_sim.mdp import state
|
|
from SkillFactory.tasks.manager_based.contact_cycle_sim.recording import SimCycleRecorder
|
|
|
|
torch.set_num_threads(4)
|
|
s = state(env)
|
|
if (s.robot.is_fixed_base or env.cfg.sim.physics.solver_type != 0
|
|
or env.cfg.sim.physics.enable_external_forces_every_iteration
|
|
or env.cfg.terminations.failed is None or env.physics_dt != .002 or env.step_dt != .01
|
|
or env.num_envs != 128 or env.cfg.episode_length_s != 9):
|
|
raise RuntimeError("Require unchanged floating-root PGS, active failures, 500Hz physics and 9s nominal task")
|
|
runtime = {"torch": str(torch.__version__), "device": str(env.device), "physics_dt": env.physics_dt,
|
|
"control_dt": env.step_dt, "num_envs": env.num_envs}
|
|
identity = {"manifest": manifest, "runtime": runtime, "seed": cfg["seeds"][0], "model_config": MODEL_CONFIG}
|
|
ledger.write_json(args.output / "runtime_identity.json", identity)
|
|
functional = json.loads((ROOT / cfg["functional_config"]).read_text())
|
|
algo = {**cfg["algorithm"], "horizon": cfg["horizon"]}
|
|
seed = cfg["seeds"][0]
|
|
torch.manual_seed(seed)
|
|
torch.cuda.manual_seed_all(seed)
|
|
model = SimResidualActorCritic(**MODEL_CONFIG).to(env.device)
|
|
initial = {k: v.detach().cpu().clone() for k, v in model.named_parameters()}
|
|
optimizer = torch.optim.Adam(model.parameters(), lr=algo["learning_rate"], eps=algo["adam_eps"])
|
|
best = None
|
|
|
|
def check_source():
|
|
guard()
|
|
if source_identity(ROOT, cfg, args.config) != manifest["sources"]:
|
|
raise RuntimeError("Bound sources/assets/config changed; refusing optimization or publication")
|
|
verify_snapshot(args.output, manifest)
|
|
if critical_log_lines(args.console_log):
|
|
raise RuntimeError("Critical simulator console log")
|
|
|
|
def save(path, update):
|
|
check_source()
|
|
remaining = cfg["max_artifact_bytes"] - artifact_bytes(args.output) - 1048576
|
|
size = save_checkpoint(path, model, optimizer, identity, update, best, ledger.data,
|
|
MODEL_CONFIG, max_bytes=remaining)
|
|
ledger.guard()
|
|
return size
|
|
|
|
@torch.no_grad()
|
|
def validation(update):
|
|
check_source()
|
|
with isolated_validation_rng(env, cfg["validation_seed"], seed):
|
|
observation = features(env.reset()[0]["policy"])
|
|
if bool(s.scenarios.any() or s.stand_only.any()):
|
|
raise RuntimeError("Validation must contain only nominal complete cycles")
|
|
returns = torch.zeros(env.num_envs, device=env.device)
|
|
live = torch.ones(env.num_envs, dtype=torch.bool, device=env.device)
|
|
with SimCycleRecorder(env) as recorder:
|
|
for step in range(900):
|
|
guard()
|
|
action, mean, _, _ = model.act(observation, deterministic=True)
|
|
recorder.before_control(observation, action, mean,
|
|
model.log_std.clamp(math.log(.02), math.log(.5)).exp())
|
|
obs, reward, terminated, truncated, _ = env.step(action)
|
|
if not torch.isfinite(reward).all():
|
|
raise RuntimeError("Nonfinite validation reward")
|
|
returns += reward * live
|
|
live &= ~(terminated | truncated)
|
|
recorder.end_control(terminated, truncated, step == 899)
|
|
observation = features(obs["policy"])
|
|
trace = {k: torch.stack([frame[k] for frame in recorder.frames]) for k in recorder.frames[0]}
|
|
# Recorded buffers are copies; release the original per-frame storage before serialization.
|
|
recorder.frames.clear()
|
|
rows = validation_rows(trace, recorder.stops, recorder.early, s.nominal, functional, evaluate)
|
|
for row, total in zip(rows, returns.cpu().tolist()):
|
|
row["episode_return"] = total
|
|
result = {"kind": ("untrained_zero_residual_network" if update == 0
|
|
else "analytic_reference_plus_learned_residual"),
|
|
"trained": update > 0, "update": update, "seed": seed,
|
|
"validation_seed": cfg["validation_seed"], "rows": rows,
|
|
"selection_key": list(selection_key(rows)), "parameter_deltas": parameter_deltas(model, initial),
|
|
"accepted_skill": False, "independent_test_executed": False,
|
|
"audit_controller_kind": "learned_policy_metadata_interface_only"}
|
|
label = f"validation_u{update:04d}"
|
|
ledger.write_json(args.output / f"{label}.json", result)
|
|
selected = ([r["env"] for r in rows if not r["success"]] or list(range(env.num_envs)))[:4]
|
|
def reserve(size):
|
|
ledger.guard(reserve_bytes=size + 1048576)
|
|
save_trace_subset(args.output / f"{label}_trace.pt", trace, selected, recorder.stops, reserve)
|
|
save_trace_subset(args.output / f"{label}_initial.pt",
|
|
{k: v[None] for k, v in recorder.initial_state.items()}, selected,
|
|
[1] * env.num_envs, reserve)
|
|
check_source()
|
|
return result
|
|
|
|
save(args.output / "untrained_u0000.pt", 0)
|
|
baseline = validation(0)
|
|
if not all(r["success"] for r in baseline["rows"]) or len(baseline["rows"]) != 128:
|
|
raise RuntimeError("u0 did not pass 128/128 functional cycles; stop before the first training rollout")
|
|
comparisons = []
|
|
x = features(env.reset()[0]["policy"])
|
|
for update in range(1, cfg["updates_per_seed"] + 1):
|
|
x, failures, timeouts, learning = optimize_reserved_rollout(
|
|
env, model, optimizer, x, algo, seed, update, ledger, guard, check_source)
|
|
if update in cfg["validation_updates"]:
|
|
row = validation(update)
|
|
if choose_candidate(best, update, row["rows"]):
|
|
name = f"candidate_u{update:04d}.pt"
|
|
best = {"update": update, "selection_key": row["selection_key"], "path": name}
|
|
save(args.output / name, update)
|
|
best["sha256"] = hashlib.sha256((args.output / name).read_bytes()).hexdigest()
|
|
comparisons.append({"update": update, "selection_key": row["selection_key"],
|
|
"parameter_deltas": row["parameter_deltas"],
|
|
"mean_height_rms_m": sum(r["height_rms_m"] for r in row["rows"]) / 128,
|
|
"mean_return": sum(r["episode_return"] for r in row["rows"]) / 128,
|
|
"mean_actual_residual_rms_rad": sum(
|
|
r["actual_residual_rms_rad"] for r in row["rows"]) / 128})
|
|
x = features(env.reset()[0]["policy"])
|
|
# Pending reservation remains charged on ANY failure; no automatic recovery.
|
|
latest_bytes = checkpoint_then_commit(
|
|
ledger, seed, update, lambda: save(args.output / "latest.pt", update))
|
|
ledger.append_json(args.output / "metrics.jsonl", {"update": update, "learner": learning,
|
|
"failures": failures, "timeouts": timeouts, "latest_checkpoint_bytes": latest_bytes,
|
|
"reserved_transitions": ledger.data["reserved_transitions"]})
|
|
check_source()
|
|
if (best is None or best["update"] == 0
|
|
or hashlib.sha256((args.output / best["path"]).read_bytes()).hexdigest() != best["sha256"]):
|
|
raise RuntimeError("Missing or changed trained candidate")
|
|
locked = {"candidate": best, "accepted_skill": False, "independent_test_executed": False,
|
|
"training_scope": "nominal_simulation_residual_pilot",
|
|
"rule": "failure fraction then all-row cost; earliest tie; never reselect from later outcomes"}
|
|
ledger.write_json(args.output / "locked_candidate.json", locked)
|
|
ledger.write_json(args.output / "baseline_comparison.json", {
|
|
"baseline_kind": baseline["kind"], "baseline_selection_key": baseline["selection_key"],
|
|
"baseline_mean_height_rms_m": sum(r["height_rms_m"] for r in baseline["rows"]) / 128,
|
|
"baseline_mean_return": sum(r["episode_return"] for r in baseline["rows"]) / 128,
|
|
"baseline_mean_actual_residual_rms_rad": sum(r["actual_residual_rms_rad"] for r in baseline["rows"]) / 128,
|
|
"updated_models": comparisons, "selection_changed_by_comparison": False,
|
|
"note": "Reference already passed at u0. Nonzero parameter deltas alone do not demonstrate improvement.",
|
|
"accepted_skill": False})
|
|
return {"passed": True, "training_complete": True, "accepted_skill": False, "locked": locked}
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
mode = parser.add_mutually_exclusive_group(required=True)
|
|
mode.add_argument("--prepare", action="store_true")
|
|
mode.add_argument("--train", action="store_true")
|
|
parser.add_argument("--config", type=Path, default=ROOT / "configs/contact_cycle/sim_training_v1.json")
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--authorization", type=Path)
|
|
parser.add_argument("--console-log", type=Path)
|
|
parser.add_argument("--supervisor-report", type=Path)
|
|
parser.add_argument("--headless", action="store_true")
|
|
parser.add_argument("--device", default="cuda:0")
|
|
args = parser.parse_args(argv)
|
|
from SkillFactory.legs.contact_cycle_sim.training import (
|
|
FreshBudgetLedger,
|
|
assert_no_conflicting_processes,
|
|
make_manifest,
|
|
prepare,
|
|
require_authorization,
|
|
validate_prepared,
|
|
writable_output,
|
|
)
|
|
|
|
args.output = writable_output(ROOT, args.output)
|
|
if args.prepare:
|
|
if args.authorization is not None or args.console_log is not None or args.supervisor_report is not None:
|
|
parser.error("Preparation is not authorization or a supervised launch")
|
|
prepare(ROOT, args.config, args.output)
|
|
print(f"Prepared only; no Kit, budget or optimizer: {args.output}", flush=True)
|
|
return
|
|
# Every gate here runs BEFORE importing Kit and BEFORE the ledger clock starts.
|
|
manifest = make_manifest(ROOT, args.config)
|
|
grant = require_authorization(args.authorization, manifest)
|
|
validate_prepared(ROOT, args.output, manifest)
|
|
if not args.headless:
|
|
raise ValueError("Pilot must be headless and externally supervised")
|
|
if (args.console_log is None or args.console_log.resolve() != args.output / "console.log"
|
|
or args.supervisor_report is None or args.supervisor_report.resolve() != args.output / "supervisor.json"
|
|
or not args.console_log.is_file() or not args.supervisor_report.is_file()):
|
|
raise ValueError("Require live supervisor console.log and supervisor.json inside artifact budget")
|
|
supervisor = json.loads(args.supervisor_report.read_text())
|
|
if supervisor.get("status") != "running" or supervisor.get("passed") is not False:
|
|
raise ValueError("Require a running external supervisor")
|
|
assert_no_conflicting_processes()
|
|
cfg = manifest["training"]
|
|
stop = {"requested": False}
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
signal.signal(sig, lambda *_: stop.update(requested=True))
|
|
app = env = safety = None
|
|
result = {"passed": False, "training_complete": False, "accepted_skill": False}
|
|
code = 1
|
|
with FreshBudgetLedger(args.output, manifest) as ledger:
|
|
def guard():
|
|
ledger.guard()
|
|
if stop["requested"]:
|
|
raise InterruptedError("Cooperative stop; pending work remains charged; no retry/resume")
|
|
if safety is not None:
|
|
safety.check()
|
|
|
|
ledger.write_json(args.output / "authorization_record.json", grant)
|
|
try:
|
|
guard()
|
|
from isaaclab.app import AppLauncher
|
|
|
|
app = AppLauncher(headless=True, device=args.device).app
|
|
guard()
|
|
from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit
|
|
from SkillFactory.tasks.manager_based.contact_cycle_sim.cfg import SimContactCycleEnvCfg
|
|
from SkillFactory.tasks.manager_based.contact_cycle_sim.env import SimContactCycleEnv
|
|
|
|
environment_cfg = SimContactCycleEnvCfg(residual_config=str(ROOT / cfg["task_config"]))
|
|
environment_cfg.scene.num_envs = cfg["num_envs"]
|
|
environment_cfg.sim.device = args.device
|
|
environment_cfg.seed = cfg["seeds"][0]
|
|
env = SimContactCycleEnv(cfg=environment_cfg)
|
|
thresholds = json.loads((ROOT / cfg["safety_config"]).read_text())["gates"]
|
|
with PhysicalSafetyAudit(env, thresholds) as safety:
|
|
result = execute(env, cfg, args, manifest, ledger, guard)
|
|
guard()
|
|
code = 0
|
|
except BaseException:
|
|
result.update(passed=False, error=traceback.format_exc())
|
|
traceback.print_exc()
|
|
finally:
|
|
if env is not None:
|
|
try:
|
|
env.close()
|
|
except BaseException:
|
|
result.update(passed=False, shutdown_error=traceback.format_exc())
|
|
code = 1
|
|
result["budget"] = ledger.data
|
|
if safety is not None:
|
|
result["physics_safety"] = safety.report()
|
|
try:
|
|
guard()
|
|
if critical_log_lines(args.console_log):
|
|
raise RuntimeError("Critical console log during shutdown")
|
|
except BaseException:
|
|
result.update(passed=False, final_guard_error=traceback.format_exc())
|
|
code = 1
|
|
result["status"] = "completed" if code == 0 else "failed"
|
|
# Never bypass the artifact/deadline guard to publish a success on failure.
|
|
try:
|
|
ledger.write_json(args.output / "result.json", result)
|
|
finally:
|
|
if app is not None:
|
|
app.close(exit_code=code)
|
|
if code:
|
|
sys.exit(code)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|