Files
SkillFactory/scripts/train_offline_leg.py
T
chenlin 093ab883be feat: release guided Go2 standing skill with explicit drift waiver
Add loaded-standing physics, PPO training, fixed-candidate acceptance, isolated inference, and fail-closed publication. Preserve the original 120-second drift failure and bind the operator-approved exception to the exact candidate and evidence.

Include pending Reach training/release infrastructure and archived packages. Leave pre-existing short-worker visualization changes and large generated training artifacts out of this commit.
2026-10-12 17:26:03 +08:00

156 lines
6.4 KiB
Python

#!/usr/bin/env python3
"""Train IQL from immutable shards, with atomic optimizer/target/RNG checkpoints."""
import argparse
import json
import os
import signal
import time
from pathlib import Path
from runtime_common import write_json
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, required=True)
parser.add_argument("--data", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--resume", type=Path)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--stop-after", type=int, help="Operational interruption drill; not a completed run")
args = parser.parse_args()
os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
import torch
from SkillFactory.legs.offline import IQL, atomic_save, canonical_hash, sha256
cfg = json.loads(args.config.read_text())
train = cfg["training"]
index = json.loads((args.data / "dataset.json").read_text())
digest = canonical_hash(cfg)
expected = cfg["collection"]["shards"] * cfg["num_envs"] * cfg["episode_steps"]
if not index["complete"] or index["config_sha256"] != digest or index["transitions"] != expected:
raise ValueError("Require complete frozen dataset matching the pre-registered config")
args.output.mkdir(parents=True, exist_ok=True)
latest = args.output / "latest.pt"
if latest.exists() and args.resume is None:
raise ValueError("Existing run requires explicit --resume; will not silently overwrite")
write_json(args.output / "result.json", {"passed": False, "status": "running", "training_complete": False})
torch.set_num_threads(4)
torch.use_deterministic_algorithms(True)
torch.set_float32_matmul_precision("highest")
torch.manual_seed(train["seed"])
data_parts = []
for record in index["shards"]:
path = args.data / record["path"]
if sha256(path) != record["sha256"]:
raise ValueError(f"Corrupted dataset: {path}")
data_parts.append(torch.load(path, weights_only=True, map_location="cpu"))
data = {key: torch.cat([part[key] for part in data_parts]).to(args.device) for key in data_parts[0]}
del data_parts
if len(data["state"]) != expected or any(not torch.isfinite(v).all() for v in data.values()):
raise ValueError("Invalid fixed dataset")
mean, std = data["state"].mean(0), data["state"].std(0).clamp(min=0.02)
learner = IQL(train, mean, std, args.device)
identity = {
"config_sha256": digest,
"dataset_sha256": sha256(args.data / "dataset.json"),
"source_sha256": {
name: sha256(Path(__file__).resolve().parents[1] / name)
for name in (
"scripts/train_offline_leg.py",
"source/SkillFactory/SkillFactory/legs/offline.py",
"source/SkillFactory/SkillFactory/legs/frozen_policy.py",
)
},
}
start_update, elapsed = 0, 0.0
if args.resume:
saved = torch.load(args.resume, map_location=args.device, weights_only=True)
if saved["identity"] != identity:
raise ValueError("Resume rejected: config/dataset/training source changed")
learner.load_state_dict(saved["learner"])
start_update, elapsed = saved["update"], saved["elapsed_s"]
started = time.perf_counter()
stop = False
def request_stop(signum, frame):
nonlocal stop
stop = True
signal.signal(signal.SIGTERM, request_stop)
signal.signal(signal.SIGINT, request_stop)
events = args.output / "events.jsonl"
with events.open("a") as stream:
stream.write(
json.dumps(
{
"event": "resume" if args.resume else "start",
"update": start_update,
"pid": os.getpid(),
"time": time.time(),
"identity": identity,
}
)
+ "\n"
)
update = start_update
for update in range(start_update + 1, train["updates"] + 1):
ids = torch.randint(expected, (train["batch_size"],), device=args.device)
losses = learner.update({key: value[ids] for key, value in data.items()})
if update % 100 == 0:
with events.open("a") as stream:
stream.write(
json.dumps(
{
"event": "update",
"update": update,
"losses": losses,
"elapsed_s": elapsed + time.perf_counter() - started,
}
)
+ "\n"
)
interrupt = stop or (args.stop_after is not None and update >= args.stop_after)
if update % train["checkpoint_every"] == 0 or interrupt or update == train["updates"]:
checkpoint = {
"learner": learner.state_dict(),
"identity": identity,
"update": update,
"elapsed_s": elapsed + time.perf_counter() - started,
}
atomic_save(checkpoint, latest)
if update % train["checkpoint_every"] == 0:
atomic_save(checkpoint, args.output / f"step_{update:06d}.pt")
print(
f"IQL update={update}/{train['updates']} elapsed={checkpoint['elapsed_s']:.1f}s losses={losses}",
flush=True,
)
if interrupt:
break
if elapsed + time.perf_counter() - started > cfg["budget"]["wall_seconds"]:
# The next update takes the ordinary interrupt/checkpoint path.
stop = True
complete = update == train["updates"]
write_json(
args.output / "result.json",
{
"passed": complete,
"training_complete": complete,
"status": "completed" if complete else "interrupted",
"updates": update,
"elapsed_s": elapsed + time.perf_counter() - started,
"identity": identity,
"checkpoint_sha256": sha256(latest),
"transitions": expected,
"device": args.device,
"acceptance_complete": False,
"algorithm": "IQL; twin-Q TD, expectile V, advantage-weighted deterministic regression",
},
)
print("Training finished; independent acceptance still required", flush=True)
if __name__ == "__main__":
main()