#!/usr/bin/env python3 """Finite shared-policy PPO prototype across asset-homogeneous Kit workers. Workers restart for each rollout in this initial integration runner. Use small update counts; this is not yet a throughput-optimized long-training service. """ import argparse import json import math import subprocess import sys import time import traceback from pathlib import Path from runtime_common import critical_log_lines, write_json def parse_args(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--bundles", type=Path, nargs="+", required=True) parser.add_argument("--num_envs", type=int, default=16, help="Per-asset environment count") parser.add_argument("--max-joints", type=int, default=8) parser.add_argument("--horizon", type=int, default=32) parser.add_argument("--updates", type=int, default=1) parser.add_argument("--episode-length", type=float, default=0.2) parser.add_argument("--seed", type=int, default=42) parser.add_argument("--device", default="cuda:0") parser.add_argument( "--viz", "--visualizer", choices=("none", "kit"), help="Worker visualization (omitted: worker default). Kit windows restart for each asset/update.", ) parser.add_argument("--output", type=Path, default=Path("logs/leg_interface/train_smoke")) args = parser.parse_args(argv) if ( min(args.num_envs, args.max_joints, args.horizon, args.updates) < 1 or not math.isfinite(args.episode_length) or args.episode_length <= 0 ): parser.error("Counts and episode length must be positive") return args def worker_command(args, bundle_path, weights, directory, seed): command = [ sys.executable, str(Path(__file__).with_name("leg_worker.py")), "--bundle", str(bundle_path.resolve()), "--mode", "rollout", "--weights", str(weights.resolve()), "--num_envs", str(args.num_envs), "--max-joints", str(args.max_joints), "--horizon", str(args.horizon), "--episode-length", str(args.episode_length), "--seed", str(seed), "--device", args.device, "--output", str(directory.resolve()), ] if args.viz is not None: command.extend(["--viz", args.viz]) return command def main(): args = parse_args() import torch from SkillFactory.legs.policy import LegActorCritic, load_policy, ppo_update, save_policy from SkillFactory.legs.spec import load_bundle from SkillFactory.legs.tokens import FEATURE_SCHEMA args.output.mkdir(parents=True, exist_ok=True) report = { "passed": False, "status": "running", "trained_skill": False, "seed": args.seed, "feature_schema": FEATURE_SCHEMA, } write_json(args.output / "result.json", report) start = time.perf_counter() try: bundles = [load_bundle(path, require_usd=True) for path in args.bundles] ids = [bundle["spec"]["part_id"] for bundle in bundles] if len(set(ids)) != len(ids) or any(len(bundle["joints"]) > args.max_joints for bundle in bundles): raise ValueError("Require unique assets fitting max-joints") timing = {(bundle["spec"]["physics_dt"], bundle["spec"]["decimation"]) for bundle in bundles} if len(timing) != 1: raise ValueError("Initial mixed trainer requires a common physics/control timestep") torch.manual_seed(args.seed) torch.set_float32_matmul_precision("highest") model = LegActorCritic().to(args.device) args.device = str(next(model.parameters()).device) optimizer = torch.optim.Adam(model.parameters(), lr=3e-4, eps=1e-5) initial = {k: v.detach().clone() for k, v in model.state_dict().items()} rows = [] for update in range(args.updates): weights = args.output / "sampling_policy.pt" save_policy(model, weights, {"update": update}) all_data, workers = [], [] for index, (bundle_path, part_id) in enumerate(zip(args.bundles, ids)): directory = args.output / f"update_{update:04d}" / part_id directory.mkdir(parents=True, exist_ok=True) log_path = directory / "worker.log" command = worker_command(args, bundle_path, weights, directory, args.seed + index + update * 100) with log_path.open("w") as log: child = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT) critical = critical_log_lines(log_path) if child.returncode or critical: raise RuntimeError(f"Worker failed ({child.returncode}): {log_path}; {critical[-3:]}") worker = json.loads((directory / "result.json").read_text()) if ( not worker["passed"] or worker["device"] != str(torch.device(args.device)) or worker["manifest_sha256"] != bundles[index]["conversion"]["manifest_sha256"] ): raise RuntimeError(f"Worker report/device/provenance mismatch: {directory}") workers.append(worker) all_data.append(torch.load(directory / "rollout.pt", weights_only=True, map_location=args.device)) # Equal per-asset environment counts yield balanced morphology sampling. data = { "observation": { key: torch.cat([item["observation"][key] for item in all_data], dim=1) for key in all_data[0]["observation"] } } for key in ("actions", "log_prob", "values", "rewards", "next_values", "done"): data[key] = torch.cat([item[key] for item in all_data], dim=1) with torch.no_grad(): sampled = {key: value.flatten(0, 1) for key, value in data["observation"].items()} current_log, _, _ = model.evaluate(sampled, data["actions"].flatten(0, 1)) mismatch = (current_log - data["log_prob"].flatten()).abs().max().item() if not math.isfinite(mismatch) or mismatch > 1e-5: raise RuntimeError(f"Sampling policy/log-prob mismatch: {mismatch}") metrics = ppo_update(model, optimizer, data) metrics["sampling_log_prob_error"] = mismatch metrics.update(update=update, workers=workers, mean_reward=data["rewards"].mean().item()) rows.append(metrics) write_json(args.output / "progress.json", rows) print( f"PPO update {update}: reward={metrics['mean_reward']:.6f}, value_loss={metrics['value_loss']:.6f}", flush=True, ) difference = max((model.state_dict()[key] - value).abs().max().item() for key, value in initial.items()) if not difference > 0 or not all(torch.isfinite(value).all() for value in model.state_dict().values()): raise RuntimeError("PPO did not produce a finite parameter update") checkpoint = args.output / "policy.pt" save_policy(model, checkpoint, {"training_assets": ids, "seed": args.seed, "updates": args.updates}) loaded = load_policy(checkpoint, args.device).eval() sample = {key: value[0] for key, value in data["observation"].items()} with torch.no_grad(): before = model.act(sample, deterministic=True) after = loaded.act(sample, deterministic=True) if not all(torch.equal(a, b) for a, b in zip(before, after)): raise RuntimeError("Checkpoint reload mismatch") report.update( passed=True, status="completed", training_assets=ids, updates=rows, environment_control_steps=args.updates * args.horizon * args.num_envs * len(ids), max_parameter_change=difference, checkpoint_reload_exact=True, wall_time_s=time.perf_counter() - start, device=args.device, checkpoint=str(checkpoint), capability="URMA-inspired goal-conditioned shared PPO smoke; not a converged/modular Skill", ) except BaseException: error = traceback.format_exc() print(error, file=sys.stderr) report.update(status="failed", error=error) write_json(args.output / "result.json", report) raise SystemExit(1) from None write_json(args.output / "result.json", report) print(f"Shared PPO integration passed: {args.output}/result.json; trained_skill=False", flush=True) if __name__ == "__main__": main()