#!/usr/bin/env python3 """Train IQL from immutable shards, with atomic optimizer/target/RNG checkpoints.""" import argparse import json import os import signal import time from pathlib import Path from runtime_common import write_json def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", type=Path, required=True) parser.add_argument("--data", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--resume", type=Path) parser.add_argument("--device", default="cuda:0") parser.add_argument("--stop-after", type=int, help="Operational interruption drill; not a completed run") args = parser.parse_args() os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") import torch from SkillFactory.legs.offline import IQL, atomic_save, canonical_hash, sha256 cfg = json.loads(args.config.read_text()) train = cfg["training"] index = json.loads((args.data / "dataset.json").read_text()) digest = canonical_hash(cfg) expected = cfg["collection"]["shards"] * cfg["num_envs"] * cfg["episode_steps"] if not index["complete"] or index["config_sha256"] != digest or index["transitions"] != expected: raise ValueError("Require complete frozen dataset matching the pre-registered config") args.output.mkdir(parents=True, exist_ok=True) latest = args.output / "latest.pt" if latest.exists() and args.resume is None: raise ValueError("Existing run requires explicit --resume; will not silently overwrite") write_json(args.output / "result.json", {"passed": False, "status": "running", "training_complete": False}) torch.set_num_threads(4) torch.use_deterministic_algorithms(True) torch.set_float32_matmul_precision("highest") torch.manual_seed(train["seed"]) data_parts = [] for record in index["shards"]: path = args.data / record["path"] if sha256(path) != record["sha256"]: raise ValueError(f"Corrupted dataset: {path}") data_parts.append(torch.load(path, weights_only=True, map_location="cpu")) data = {key: torch.cat([part[key] for part in data_parts]).to(args.device) for key in data_parts[0]} del data_parts if len(data["state"]) != expected or any(not torch.isfinite(v).all() for v in data.values()): raise ValueError("Invalid fixed dataset") mean, std = data["state"].mean(0), data["state"].std(0).clamp(min=0.02) learner = IQL(train, mean, std, args.device) identity = { "config_sha256": digest, "dataset_sha256": sha256(args.data / "dataset.json"), "source_sha256": { name: sha256(Path(__file__).resolve().parents[1] / name) for name in ( "scripts/train_offline_leg.py", "source/SkillFactory/SkillFactory/legs/offline.py", "source/SkillFactory/SkillFactory/legs/frozen_policy.py", ) }, } start_update, elapsed = 0, 0.0 if args.resume: saved = torch.load(args.resume, map_location=args.device, weights_only=True) if saved["identity"] != identity: raise ValueError("Resume rejected: config/dataset/training source changed") learner.load_state_dict(saved["learner"]) start_update, elapsed = saved["update"], saved["elapsed_s"] started = time.perf_counter() stop = False def request_stop(signum, frame): nonlocal stop stop = True signal.signal(signal.SIGTERM, request_stop) signal.signal(signal.SIGINT, request_stop) events = args.output / "events.jsonl" with events.open("a") as stream: stream.write( json.dumps( { "event": "resume" if args.resume else "start", "update": start_update, "pid": os.getpid(), "time": time.time(), "identity": identity, } ) + "\n" ) update = start_update for update in range(start_update + 1, train["updates"] + 1): ids = torch.randint(expected, (train["batch_size"],), device=args.device) losses = learner.update({key: value[ids] for key, value in data.items()}) if update % 100 == 0: with events.open("a") as stream: stream.write( json.dumps( { "event": "update", "update": update, "losses": losses, "elapsed_s": elapsed + time.perf_counter() - started, } ) + "\n" ) interrupt = stop or (args.stop_after is not None and update >= args.stop_after) if update % train["checkpoint_every"] == 0 or interrupt or update == train["updates"]: checkpoint = { "learner": learner.state_dict(), "identity": identity, "update": update, "elapsed_s": elapsed + time.perf_counter() - started, } atomic_save(checkpoint, latest) if update % train["checkpoint_every"] == 0: atomic_save(checkpoint, args.output / f"step_{update:06d}.pt") print( f"IQL update={update}/{train['updates']} elapsed={checkpoint['elapsed_s']:.1f}s losses={losses}", flush=True, ) if interrupt: break if elapsed + time.perf_counter() - started > cfg["budget"]["wall_seconds"]: # The next update takes the ordinary interrupt/checkpoint path. stop = True complete = update == train["updates"] write_json( args.output / "result.json", { "passed": complete, "training_complete": complete, "status": "completed" if complete else "interrupted", "updates": update, "elapsed_s": elapsed + time.perf_counter() - started, "identity": identity, "checkpoint_sha256": sha256(latest), "transitions": expected, "device": args.device, "acceptance_complete": False, "algorithm": "IQL; twin-Q TD, expectile V, advantage-weighted deterministic regression", }, ) print("Training finished; independent acceptance still required", flush=True) if __name__ == "__main__": main()