9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
193 lines
8.6 KiB
Python
193 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Finite shared-policy PPO prototype across asset-homogeneous Kit workers.
|
|
|
|
Workers restart for each rollout in this initial integration runner. Use small
|
|
update counts; this is not yet a throughput-optimized long-training service.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
from runtime_common import critical_log_lines, write_json
|
|
|
|
|
|
def parse_args(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--bundles", type=Path, nargs="+", required=True)
|
|
parser.add_argument("--num_envs", type=int, default=16, help="Per-asset environment count")
|
|
parser.add_argument("--max-joints", type=int, default=8)
|
|
parser.add_argument("--horizon", type=int, default=32)
|
|
parser.add_argument("--updates", type=int, default=1)
|
|
parser.add_argument("--episode-length", type=float, default=0.2)
|
|
parser.add_argument("--seed", type=int, default=42)
|
|
parser.add_argument("--device", default="cuda:0")
|
|
parser.add_argument(
|
|
"--viz",
|
|
"--visualizer",
|
|
choices=("none", "kit"),
|
|
help="Worker visualization (omitted: worker default). Kit windows restart for each asset/update.",
|
|
)
|
|
parser.add_argument("--output", type=Path, default=Path("logs/leg_interface/train_smoke"))
|
|
args = parser.parse_args(argv)
|
|
if (
|
|
min(args.num_envs, args.max_joints, args.horizon, args.updates) < 1
|
|
or not math.isfinite(args.episode_length)
|
|
or args.episode_length <= 0
|
|
):
|
|
parser.error("Counts and episode length must be positive")
|
|
return args
|
|
|
|
|
|
def worker_command(args, bundle_path, weights, directory, seed):
|
|
command = [
|
|
sys.executable,
|
|
str(Path(__file__).with_name("leg_worker.py")),
|
|
"--bundle",
|
|
str(bundle_path.resolve()),
|
|
"--mode",
|
|
"rollout",
|
|
"--weights",
|
|
str(weights.resolve()),
|
|
"--num_envs",
|
|
str(args.num_envs),
|
|
"--max-joints",
|
|
str(args.max_joints),
|
|
"--horizon",
|
|
str(args.horizon),
|
|
"--episode-length",
|
|
str(args.episode_length),
|
|
"--seed",
|
|
str(seed),
|
|
"--device",
|
|
args.device,
|
|
"--output",
|
|
str(directory.resolve()),
|
|
]
|
|
if args.viz is not None:
|
|
command.extend(["--viz", args.viz])
|
|
return command
|
|
|
|
|
|
def main():
|
|
args = parse_args()
|
|
import torch
|
|
from SkillFactory.legs.policy import LegActorCritic, load_policy, ppo_update, save_policy
|
|
from SkillFactory.legs.spec import load_bundle
|
|
from SkillFactory.legs.tokens import FEATURE_SCHEMA
|
|
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
report = {
|
|
"passed": False,
|
|
"status": "running",
|
|
"trained_skill": False,
|
|
"seed": args.seed,
|
|
"feature_schema": FEATURE_SCHEMA,
|
|
}
|
|
write_json(args.output / "result.json", report)
|
|
start = time.perf_counter()
|
|
try:
|
|
bundles = [load_bundle(path, require_usd=True) for path in args.bundles]
|
|
ids = [bundle["spec"]["part_id"] for bundle in bundles]
|
|
if len(set(ids)) != len(ids) or any(len(bundle["joints"]) > args.max_joints for bundle in bundles):
|
|
raise ValueError("Require unique assets fitting max-joints")
|
|
timing = {(bundle["spec"]["physics_dt"], bundle["spec"]["decimation"]) for bundle in bundles}
|
|
if len(timing) != 1:
|
|
raise ValueError("Initial mixed trainer requires a common physics/control timestep")
|
|
torch.manual_seed(args.seed)
|
|
torch.set_float32_matmul_precision("highest")
|
|
model = LegActorCritic().to(args.device)
|
|
args.device = str(next(model.parameters()).device)
|
|
optimizer = torch.optim.Adam(model.parameters(), lr=3e-4, eps=1e-5)
|
|
initial = {k: v.detach().clone() for k, v in model.state_dict().items()}
|
|
rows = []
|
|
for update in range(args.updates):
|
|
weights = args.output / "sampling_policy.pt"
|
|
save_policy(model, weights, {"update": update})
|
|
all_data, workers = [], []
|
|
for index, (bundle_path, part_id) in enumerate(zip(args.bundles, ids)):
|
|
directory = args.output / f"update_{update:04d}" / part_id
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
log_path = directory / "worker.log"
|
|
command = worker_command(args, bundle_path, weights, directory, args.seed + index + update * 100)
|
|
with log_path.open("w") as log:
|
|
child = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT)
|
|
critical = critical_log_lines(log_path)
|
|
if child.returncode or critical:
|
|
raise RuntimeError(f"Worker failed ({child.returncode}): {log_path}; {critical[-3:]}")
|
|
worker = json.loads((directory / "result.json").read_text())
|
|
if (
|
|
not worker["passed"]
|
|
or worker["device"] != str(torch.device(args.device))
|
|
or worker["manifest_sha256"] != bundles[index]["conversion"]["manifest_sha256"]
|
|
):
|
|
raise RuntimeError(f"Worker report/device/provenance mismatch: {directory}")
|
|
workers.append(worker)
|
|
all_data.append(torch.load(directory / "rollout.pt", weights_only=True, map_location=args.device))
|
|
# Equal per-asset environment counts yield balanced morphology sampling.
|
|
data = {
|
|
"observation": {
|
|
key: torch.cat([item["observation"][key] for item in all_data], dim=1)
|
|
for key in all_data[0]["observation"]
|
|
}
|
|
}
|
|
for key in ("actions", "log_prob", "values", "rewards", "next_values", "done"):
|
|
data[key] = torch.cat([item[key] for item in all_data], dim=1)
|
|
with torch.no_grad():
|
|
sampled = {key: value.flatten(0, 1) for key, value in data["observation"].items()}
|
|
current_log, _, _ = model.evaluate(sampled, data["actions"].flatten(0, 1))
|
|
mismatch = (current_log - data["log_prob"].flatten()).abs().max().item()
|
|
if not math.isfinite(mismatch) or mismatch > 1e-5:
|
|
raise RuntimeError(f"Sampling policy/log-prob mismatch: {mismatch}")
|
|
metrics = ppo_update(model, optimizer, data)
|
|
metrics["sampling_log_prob_error"] = mismatch
|
|
metrics.update(update=update, workers=workers, mean_reward=data["rewards"].mean().item())
|
|
rows.append(metrics)
|
|
write_json(args.output / "progress.json", rows)
|
|
print(
|
|
f"PPO update {update}: reward={metrics['mean_reward']:.6f}, value_loss={metrics['value_loss']:.6f}",
|
|
flush=True,
|
|
)
|
|
difference = max((model.state_dict()[key] - value).abs().max().item() for key, value in initial.items())
|
|
if not difference > 0 or not all(torch.isfinite(value).all() for value in model.state_dict().values()):
|
|
raise RuntimeError("PPO did not produce a finite parameter update")
|
|
checkpoint = args.output / "policy.pt"
|
|
save_policy(model, checkpoint, {"training_assets": ids, "seed": args.seed, "updates": args.updates})
|
|
loaded = load_policy(checkpoint, args.device).eval()
|
|
sample = {key: value[0] for key, value in data["observation"].items()}
|
|
with torch.no_grad():
|
|
before = model.act(sample, deterministic=True)
|
|
after = loaded.act(sample, deterministic=True)
|
|
if not all(torch.equal(a, b) for a, b in zip(before, after)):
|
|
raise RuntimeError("Checkpoint reload mismatch")
|
|
report.update(
|
|
passed=True,
|
|
status="completed",
|
|
training_assets=ids,
|
|
updates=rows,
|
|
environment_control_steps=args.updates * args.horizon * args.num_envs * len(ids),
|
|
max_parameter_change=difference,
|
|
checkpoint_reload_exact=True,
|
|
wall_time_s=time.perf_counter() - start,
|
|
device=args.device,
|
|
checkpoint=str(checkpoint),
|
|
capability="URMA-inspired goal-conditioned shared PPO smoke; not a converged/modular Skill",
|
|
)
|
|
except BaseException:
|
|
error = traceback.format_exc()
|
|
print(error, file=sys.stderr)
|
|
report.update(status="failed", error=error)
|
|
write_json(args.output / "result.json", report)
|
|
raise SystemExit(1) from None
|
|
write_json(args.output / "result.json", report)
|
|
print(f"Shared PPO integration passed: {args.output}/result.json; trained_skill=False", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|