093ab883be
Add loaded-standing physics, PPO training, fixed-candidate acceptance, isolated inference, and fail-closed publication. Preserve the original 120-second drift failure and bind the operator-approved exception to the exact candidate and evidence. Include pending Reach training/release infrastructure and archived packages. Leave pre-existing short-worker visualization changes and large generated training artifacts out of this commit.
156 lines
6.4 KiB
Python
156 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Train IQL from immutable shards, with atomic optimizer/target/RNG checkpoints."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import signal
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from runtime_common import write_json
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--config", type=Path, required=True)
|
|
parser.add_argument("--data", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--resume", type=Path)
|
|
parser.add_argument("--device", default="cuda:0")
|
|
parser.add_argument("--stop-after", type=int, help="Operational interruption drill; not a completed run")
|
|
args = parser.parse_args()
|
|
os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
|
|
import torch
|
|
from SkillFactory.legs.offline import IQL, atomic_save, canonical_hash, sha256
|
|
|
|
cfg = json.loads(args.config.read_text())
|
|
train = cfg["training"]
|
|
index = json.loads((args.data / "dataset.json").read_text())
|
|
digest = canonical_hash(cfg)
|
|
expected = cfg["collection"]["shards"] * cfg["num_envs"] * cfg["episode_steps"]
|
|
if not index["complete"] or index["config_sha256"] != digest or index["transitions"] != expected:
|
|
raise ValueError("Require complete frozen dataset matching the pre-registered config")
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
latest = args.output / "latest.pt"
|
|
if latest.exists() and args.resume is None:
|
|
raise ValueError("Existing run requires explicit --resume; will not silently overwrite")
|
|
write_json(args.output / "result.json", {"passed": False, "status": "running", "training_complete": False})
|
|
torch.set_num_threads(4)
|
|
torch.use_deterministic_algorithms(True)
|
|
torch.set_float32_matmul_precision("highest")
|
|
torch.manual_seed(train["seed"])
|
|
data_parts = []
|
|
for record in index["shards"]:
|
|
path = args.data / record["path"]
|
|
if sha256(path) != record["sha256"]:
|
|
raise ValueError(f"Corrupted dataset: {path}")
|
|
data_parts.append(torch.load(path, weights_only=True, map_location="cpu"))
|
|
data = {key: torch.cat([part[key] for part in data_parts]).to(args.device) for key in data_parts[0]}
|
|
del data_parts
|
|
if len(data["state"]) != expected or any(not torch.isfinite(v).all() for v in data.values()):
|
|
raise ValueError("Invalid fixed dataset")
|
|
mean, std = data["state"].mean(0), data["state"].std(0).clamp(min=0.02)
|
|
learner = IQL(train, mean, std, args.device)
|
|
identity = {
|
|
"config_sha256": digest,
|
|
"dataset_sha256": sha256(args.data / "dataset.json"),
|
|
"source_sha256": {
|
|
name: sha256(Path(__file__).resolve().parents[1] / name)
|
|
for name in (
|
|
"scripts/train_offline_leg.py",
|
|
"source/SkillFactory/SkillFactory/legs/offline.py",
|
|
"source/SkillFactory/SkillFactory/legs/frozen_policy.py",
|
|
)
|
|
},
|
|
}
|
|
start_update, elapsed = 0, 0.0
|
|
if args.resume:
|
|
saved = torch.load(args.resume, map_location=args.device, weights_only=True)
|
|
if saved["identity"] != identity:
|
|
raise ValueError("Resume rejected: config/dataset/training source changed")
|
|
learner.load_state_dict(saved["learner"])
|
|
start_update, elapsed = saved["update"], saved["elapsed_s"]
|
|
started = time.perf_counter()
|
|
stop = False
|
|
|
|
def request_stop(signum, frame):
|
|
nonlocal stop
|
|
stop = True
|
|
|
|
signal.signal(signal.SIGTERM, request_stop)
|
|
signal.signal(signal.SIGINT, request_stop)
|
|
events = args.output / "events.jsonl"
|
|
with events.open("a") as stream:
|
|
stream.write(
|
|
json.dumps(
|
|
{
|
|
"event": "resume" if args.resume else "start",
|
|
"update": start_update,
|
|
"pid": os.getpid(),
|
|
"time": time.time(),
|
|
"identity": identity,
|
|
}
|
|
)
|
|
+ "\n"
|
|
)
|
|
update = start_update
|
|
for update in range(start_update + 1, train["updates"] + 1):
|
|
ids = torch.randint(expected, (train["batch_size"],), device=args.device)
|
|
losses = learner.update({key: value[ids] for key, value in data.items()})
|
|
if update % 100 == 0:
|
|
with events.open("a") as stream:
|
|
stream.write(
|
|
json.dumps(
|
|
{
|
|
"event": "update",
|
|
"update": update,
|
|
"losses": losses,
|
|
"elapsed_s": elapsed + time.perf_counter() - started,
|
|
}
|
|
)
|
|
+ "\n"
|
|
)
|
|
interrupt = stop or (args.stop_after is not None and update >= args.stop_after)
|
|
if update % train["checkpoint_every"] == 0 or interrupt or update == train["updates"]:
|
|
checkpoint = {
|
|
"learner": learner.state_dict(),
|
|
"identity": identity,
|
|
"update": update,
|
|
"elapsed_s": elapsed + time.perf_counter() - started,
|
|
}
|
|
atomic_save(checkpoint, latest)
|
|
if update % train["checkpoint_every"] == 0:
|
|
atomic_save(checkpoint, args.output / f"step_{update:06d}.pt")
|
|
print(
|
|
f"IQL update={update}/{train['updates']} elapsed={checkpoint['elapsed_s']:.1f}s losses={losses}",
|
|
flush=True,
|
|
)
|
|
if interrupt:
|
|
break
|
|
if elapsed + time.perf_counter() - started > cfg["budget"]["wall_seconds"]:
|
|
# The next update takes the ordinary interrupt/checkpoint path.
|
|
stop = True
|
|
complete = update == train["updates"]
|
|
write_json(
|
|
args.output / "result.json",
|
|
{
|
|
"passed": complete,
|
|
"training_complete": complete,
|
|
"status": "completed" if complete else "interrupted",
|
|
"updates": update,
|
|
"elapsed_s": elapsed + time.perf_counter() - started,
|
|
"identity": identity,
|
|
"checkpoint_sha256": sha256(latest),
|
|
"transitions": expected,
|
|
"device": args.device,
|
|
"acceptance_complete": False,
|
|
"algorithm": "IQL; twin-Q TD, expectile V, advantage-weighted deterministic regression",
|
|
},
|
|
)
|
|
print("Training finished; independent acceptance still required", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|