Files
SkillFactory/scripts/train_leg_skill.py
T
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

193 lines
8.6 KiB
Python

#!/usr/bin/env python3
"""Finite shared-policy PPO prototype across asset-homogeneous Kit workers.
Workers restart for each rollout in this initial integration runner. Use small
update counts; this is not yet a throughput-optimized long-training service.
"""
import argparse
import json
import math
import subprocess
import sys
import time
import traceback
from pathlib import Path
from runtime_common import critical_log_lines, write_json
def parse_args(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--bundles", type=Path, nargs="+", required=True)
parser.add_argument("--num_envs", type=int, default=16, help="Per-asset environment count")
parser.add_argument("--max-joints", type=int, default=8)
parser.add_argument("--horizon", type=int, default=32)
parser.add_argument("--updates", type=int, default=1)
parser.add_argument("--episode-length", type=float, default=0.2)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--device", default="cuda:0")
parser.add_argument(
"--viz",
"--visualizer",
choices=("none", "kit"),
help="Worker visualization (omitted: worker default). Kit windows restart for each asset/update.",
)
parser.add_argument("--output", type=Path, default=Path("logs/leg_interface/train_smoke"))
args = parser.parse_args(argv)
if (
min(args.num_envs, args.max_joints, args.horizon, args.updates) < 1
or not math.isfinite(args.episode_length)
or args.episode_length <= 0
):
parser.error("Counts and episode length must be positive")
return args
def worker_command(args, bundle_path, weights, directory, seed):
command = [
sys.executable,
str(Path(__file__).with_name("leg_worker.py")),
"--bundle",
str(bundle_path.resolve()),
"--mode",
"rollout",
"--weights",
str(weights.resolve()),
"--num_envs",
str(args.num_envs),
"--max-joints",
str(args.max_joints),
"--horizon",
str(args.horizon),
"--episode-length",
str(args.episode_length),
"--seed",
str(seed),
"--device",
args.device,
"--output",
str(directory.resolve()),
]
if args.viz is not None:
command.extend(["--viz", args.viz])
return command
def main():
args = parse_args()
import torch
from SkillFactory.legs.policy import LegActorCritic, load_policy, ppo_update, save_policy
from SkillFactory.legs.spec import load_bundle
from SkillFactory.legs.tokens import FEATURE_SCHEMA
args.output.mkdir(parents=True, exist_ok=True)
report = {
"passed": False,
"status": "running",
"trained_skill": False,
"seed": args.seed,
"feature_schema": FEATURE_SCHEMA,
}
write_json(args.output / "result.json", report)
start = time.perf_counter()
try:
bundles = [load_bundle(path, require_usd=True) for path in args.bundles]
ids = [bundle["spec"]["part_id"] for bundle in bundles]
if len(set(ids)) != len(ids) or any(len(bundle["joints"]) > args.max_joints for bundle in bundles):
raise ValueError("Require unique assets fitting max-joints")
timing = {(bundle["spec"]["physics_dt"], bundle["spec"]["decimation"]) for bundle in bundles}
if len(timing) != 1:
raise ValueError("Initial mixed trainer requires a common physics/control timestep")
torch.manual_seed(args.seed)
torch.set_float32_matmul_precision("highest")
model = LegActorCritic().to(args.device)
args.device = str(next(model.parameters()).device)
optimizer = torch.optim.Adam(model.parameters(), lr=3e-4, eps=1e-5)
initial = {k: v.detach().clone() for k, v in model.state_dict().items()}
rows = []
for update in range(args.updates):
weights = args.output / "sampling_policy.pt"
save_policy(model, weights, {"update": update})
all_data, workers = [], []
for index, (bundle_path, part_id) in enumerate(zip(args.bundles, ids)):
directory = args.output / f"update_{update:04d}" / part_id
directory.mkdir(parents=True, exist_ok=True)
log_path = directory / "worker.log"
command = worker_command(args, bundle_path, weights, directory, args.seed + index + update * 100)
with log_path.open("w") as log:
child = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT)
critical = critical_log_lines(log_path)
if child.returncode or critical:
raise RuntimeError(f"Worker failed ({child.returncode}): {log_path}; {critical[-3:]}")
worker = json.loads((directory / "result.json").read_text())
if (
not worker["passed"]
or worker["device"] != str(torch.device(args.device))
or worker["manifest_sha256"] != bundles[index]["conversion"]["manifest_sha256"]
):
raise RuntimeError(f"Worker report/device/provenance mismatch: {directory}")
workers.append(worker)
all_data.append(torch.load(directory / "rollout.pt", weights_only=True, map_location=args.device))
# Equal per-asset environment counts yield balanced morphology sampling.
data = {
"observation": {
key: torch.cat([item["observation"][key] for item in all_data], dim=1)
for key in all_data[0]["observation"]
}
}
for key in ("actions", "log_prob", "values", "rewards", "next_values", "done"):
data[key] = torch.cat([item[key] for item in all_data], dim=1)
with torch.no_grad():
sampled = {key: value.flatten(0, 1) for key, value in data["observation"].items()}
current_log, _, _ = model.evaluate(sampled, data["actions"].flatten(0, 1))
mismatch = (current_log - data["log_prob"].flatten()).abs().max().item()
if not math.isfinite(mismatch) or mismatch > 1e-5:
raise RuntimeError(f"Sampling policy/log-prob mismatch: {mismatch}")
metrics = ppo_update(model, optimizer, data)
metrics["sampling_log_prob_error"] = mismatch
metrics.update(update=update, workers=workers, mean_reward=data["rewards"].mean().item())
rows.append(metrics)
write_json(args.output / "progress.json", rows)
print(
f"PPO update {update}: reward={metrics['mean_reward']:.6f}, value_loss={metrics['value_loss']:.6f}",
flush=True,
)
difference = max((model.state_dict()[key] - value).abs().max().item() for key, value in initial.items())
if not difference > 0 or not all(torch.isfinite(value).all() for value in model.state_dict().values()):
raise RuntimeError("PPO did not produce a finite parameter update")
checkpoint = args.output / "policy.pt"
save_policy(model, checkpoint, {"training_assets": ids, "seed": args.seed, "updates": args.updates})
loaded = load_policy(checkpoint, args.device).eval()
sample = {key: value[0] for key, value in data["observation"].items()}
with torch.no_grad():
before = model.act(sample, deterministic=True)
after = loaded.act(sample, deterministic=True)
if not all(torch.equal(a, b) for a, b in zip(before, after)):
raise RuntimeError("Checkpoint reload mismatch")
report.update(
passed=True,
status="completed",
training_assets=ids,
updates=rows,
environment_control_steps=args.updates * args.horizon * args.num_envs * len(ids),
max_parameter_change=difference,
checkpoint_reload_exact=True,
wall_time_s=time.perf_counter() - start,
device=args.device,
checkpoint=str(checkpoint),
capability="URMA-inspired goal-conditioned shared PPO smoke; not a converged/modular Skill",
)
except BaseException:
error = traceback.format_exc()
print(error, file=sys.stderr)
report.update(status="failed", error=error)
write_json(args.output / "result.json", report)
raise SystemExit(1) from None
write_json(args.output / "result.json", report)
print(f"Shared PPO integration passed: {args.output}/result.json; trained_skill=False", flush=True)
if __name__ == "__main__":
main()