Files
SkillFactory/scripts/rootxy_launch_v2.py
T
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

872 lines
48 KiB
Python

"""Dedicated RootXY supervisor adapter. Importing this module never starts Kit.
Ownership/native monitoring/launch transport reuse reviewed evaluator helpers;
no evaluation grant, protocol, ledger, checkpoint or candidate authority is reused.
"""
import argparse
import ctypes
from contextlib import contextmanager
import fcntl
import hashlib
import importlib
import json
import math
import os
import signal
import socket
import struct
import subprocess
import sys
import time
from pathlib import Path
from types import SimpleNamespace
import evaluate_cycle_sim_residual as owner
ROOT = owner.ROOT
SCRIPT = 'scripts/train_cycle_sim_rootxy_v2.py'
PROFILE = 'rootxy-v2-7200s-1GiB-73460calls'
TASK_ID = 'SkillFactory-Go2FL-ContactCycleSimRootXY-v2'
PREREG_FORMAT = 'skillfactory.contact-cycle-sim-rootxy-preregistration.v2'
GRANT_FORMAT = 'skillfactory.rootxy-parent-grant.v2'
POLICY_FORMAT = 'skillfactory.rootxy-native-policy.v2'
PREPARATION_FORMAT = 'skillfactory.rootxy-config-preparation.v2'
STORAGE_SCOPE = 'rootxy-new-owned.native-monitored-stop.v2'
DECISION_FORMAT = 'skillfactory.rootxy-p2-decision.v2'
CAP = 1 << 30
HEADROOM = 16 << 20
STDOUT = 1 << 20
NATIVE_LIFETIME = 10800
b = owner.cpu_module('evaluation_budget')
def canonical(value):
return b.encode(value)
def sha(value):
return hashlib.sha256(canonical(value)).hexdigest()
def read(path, expected=None):
path = Path(path)
if not path.is_file() or path.is_symlink() or path.stat().st_size > 32 << 20:
raise ValueError('Missing/unsafe/oversized RootXY input: ' + str(path))
data = path.read_bytes()
if expected is not None:
owner.read_trusted(path, expected)
return json.loads(data)
def checked_root(path):
root = Path(path)
if (not root.is_absolute() or root != root.resolve()
or root.is_relative_to(ROOT) and not root.is_relative_to(ROOT / 'logs/contact_cycle/rootxy_v2')
or root.is_relative_to(ROOT / 'releases')
or root.is_relative_to(ROOT / 'logs/contact_cycle/sim_residual_training_v1')):
raise ValueError('Canonical new isolated RootXY root required')
return root
def verify_sources(manifest):
if (manifest.get('format') != PREREG_FORMAT or manifest.get('task_id') != TASK_ID
or manifest.get('launch_integration_complete') is not True
or manifest.get('accepted_skill') is not False):
raise ValueError('New RootXY preregistration required')
for relative, expected in manifest['sources'].items():
path = ROOT / relative
if (Path(relative).is_absolute() or '..' in Path(relative).parts or path != path.resolve()
or b.digest(path) != expected):
raise ValueError('RootXY source drift: ' + relative)
cfg = manifest['training']
fixed = {'seeds': [92511], 'validation_seed': 93511, 'num_envs': 128,
'max_total_wall_s': 7200, 'cleanup_reserve_s': 120, 'max_artifact_bytes': CAP,
'max_validation_transitions': 806400, 'validation_updates': [64, 128, 192, 256]}
if any(cfg.get(k) != v for k, v in fixed.items()):
raise ValueError('RootXY resource contract changed')
def launch_identity():
app_path = Path('/home/cen/IsaacLab/source/isaaclab/isaaclab/app/app_launcher.py')
return {'app_launcher': {'path': str(app_path), 'sha256': b.digest(app_path)}, 'format': 'skillfactory.rootxy-launch-identity.v2',
'transport': owner.launch_identity(), 'profile': PROFILE, 'task_id': TASK_ID,
'entrypoint': {'path': str(ROOT / SCRIPT), 'sha256': b.digest(ROOT / SCRIPT)},
'adapter': {'path': str(Path(__file__).resolve()), 'sha256': b.digest(__file__)}}
def redirects(root, mode):
return owner.native_redirects(root / 'native' / mode, preparation=True)
def policy_check(args, now):
policy = read(args.storage_policy, args.trusted_storage_policy_sha256)
root = args.output
fields = {'format', 'profile', 'artifact_root', 'limit_bytes', 'native_paths', 'shared_native_paths',
'task_owned_cache_paths', 'free_space_paths', 'free_space_mount_identities', 'mount_identity',
'check_interval_seconds', 'max_native_growth_bytes', 'min_free_bytes', 'coverage_gaps',
'native_hard_cap', 'boot_id', 'issued_boottime', 'not_after_boottime', 'lifetime_seconds'}
duration = policy.get('lifetime_seconds')
if (set(policy) != fields or policy['format'] != POLICY_FORMAT or policy['profile'] != PROFILE
or policy['artifact_root'] != str(root) or policy['limit_bytes'] != CAP
or policy['native_paths'] != owner.native_paths(root)
or policy['shared_native_paths'] != owner.shared_native_paths()
or policy['task_owned_cache_paths'] != sorted([*redirects(root, 'prepare').values(),
*redirects(root, 'run').values()])
or policy['free_space_paths'] != owner.free_space_paths(root)
or policy['free_space_mount_identities'] != {
p: owner.mount_identity(Path(p)) for p in owner.free_space_paths(root)}
or policy['mount_identity'] != owner.mount_identity(root)
or policy['coverage_gaps'] != list(owner.NATIVE_GAPS) or policy['native_hard_cap'] is not False
or policy['boot_id'] != b.boot_id() or type(duration) is not int or not 7200 <= duration <= 10800
or type(policy['issued_boottime']) not in (int, float)
or not 0 <= policy['issued_boottime'] <= now
or policy['not_after_boottime'] != policy['issued_boottime'] + duration
or type(policy['check_interval_seconds']) not in (int, float)
or not 0 < policy['check_interval_seconds'] <= 1
or type(policy['max_native_growth_bytes']) is not int
or not 0 < policy['max_native_growth_bytes'] <= CAP
or type(policy['min_free_bytes']) is not int or policy['min_free_bytes'] < 2 * CAP):
raise PermissionError('Exact RootXY native MONITOR policy required (not quota)')
return policy
def admission(args, now):
args.output = checked_root(args.output)
mode = 'prepare' if args.prepare else 'run'
required = ('manifest', 'trusted_manifest_sha256', 'authorization', 'trusted_authorization_sha256',
'storage_policy', 'trusted_storage_policy_sha256')
if any(getattr(args, k, None) is None for k in required):
raise PermissionError('Fresh exact RootXY manifest/native-policy/parent grant required')
manifest = read(args.manifest, args.trusted_manifest_sha256)
verify_sources(manifest)
grant = read(args.authorization, args.trusted_authorization_sha256)
fields = {'format', 'profile', 'task_id', 'mode', 'artifact_root', 'manifest_sha256', 'storage_policy_sha256',
'launch_identity_sha256', 'boot_id', 'not_after_boottime', 'parent_review',
'preparation_sha256', 'binding_sha256', 'p2_decision_sha256', 'gpu_identity'}
if (set(grant) != fields or grant['format'] != GRANT_FORMAT or grant['profile'] != PROFILE
or grant['task_id'] != TASK_ID or grant['mode'] != mode or grant['artifact_root'] != str(args.output)
or grant['manifest_sha256'] != args.trusted_manifest_sha256
or grant['storage_policy_sha256'] != args.trusted_storage_policy_sha256
or grant['launch_identity_sha256'] != sha(launch_identity())
or grant['boot_id'] != b.boot_id() or type(grant['not_after_boottime']) not in (int, float)
or not now < grant['not_after_boottime'] <= now + 3600
or type(grant['parent_review']) is not str or not grant['parent_review'].strip()):
raise PermissionError('RootXY parent grant identity/expiry mismatch')
if mode == 'prepare':
if any(grant[k] is not None for k in ('preparation_sha256', 'binding_sha256', 'p2_decision_sha256')):
raise PermissionError('PREP has pending P2/config identity only, not runtime approval')
if args.output.exists():
raise FileExistsError('PREP requires a NEW root; no retry')
else:
for field in ('preparation_sha256', 'binding_sha256', 'p2_decision_sha256'):
value = grant[field]
if (type(value) is not str or len(value) != 64
or any(c not in '0123456789abcdef' for c in value)):
raise PermissionError('RUN requires an exact independently approved ' + field)
if any((args.output / name).exists() for name in (
'run-claim.lock', 'launch_claim.json', 'budget.json', 'budget.lock', 'locked_candidate.json',
'run-worker-result.json', 'run-supervisor.json')):
raise FileExistsError('Fresh-only RootXY RUN; no prior trial or candidate artifacts')
preparation = read(args.output / 'preparation.json', grant['preparation_sha256'])
binding = read(args.output / 'training-binding.json', grant['binding_sha256'])
decision = read(args.p2_decision, grant['p2_decision_sha256'])
if (preparation.get('format') != PREPARATION_FORMAT or preparation.get('status') != 'prepared_config_only'
or preparation.get('manifest_sha256') != args.trusted_manifest_sha256
or preparation.get('binding_sha256') != grant['binding_sha256']
or preparation.get('artifact_root') != str(args.output)
or preparation.get('physics_calls') != 0 or preparation.get('runtime_qualified') is not False
or binding.get('format') != 'skillfactory.rootxy-training-binding.v2'
or binding.get('artifact_root') != str(args.output) or binding.get('profile') != PROFILE):
raise PermissionError('Exact config-only preparation/binding required')
prep_supervisor = read(args.output / 'prepare-supervisor.json', preparation['supervisor_sha256'])
read(args.output / 'prepare-worker-result.json', preparation['worker_result_sha256'])
if (preparation.get('postexit_verified') is not True
or prep_supervisor.get('success') is not True or prep_supervisor.get('cleanup_clean') is not True):
raise PermissionError('PREP needs independent clean waited-exit completion')
if (set(decision) != {'format', 'task_id', 'artifact_root', 'manifest_sha256', 'preparation_sha256',
'binding_sha256', 'approved', 'parent_review', 'p2_evidence'}
or decision['format'] != DECISION_FORMAT or decision['task_id'] != TASK_ID
or decision['artifact_root'] != str(args.output) or decision['approved'] is not True
or decision['manifest_sha256'] != args.trusted_manifest_sha256
or decision['preparation_sha256'] != grant['preparation_sha256']
or decision['binding_sha256'] != grant['binding_sha256']
or not isinstance(decision['parent_review'], str) or not decision['parent_review'].strip()
or not isinstance(decision['p2_evidence'], dict) or not decision['p2_evidence']):
raise PermissionError('Separate parent-approved P2 evidence/feasibility decision required')
for path, digest in decision['p2_evidence'].items():
owner.read_trusted(path, digest)
for relative, digest in manifest['sources'].items():
if b.digest(args.output / 'source_snapshot' / relative) != digest:
raise ValueError('RootXY preparation snapshot drift')
policy = policy_check(args, now)
total = 600 if mode == 'prepare' else 7200
if policy['not_after_boottime'] < max(grant['not_after_boottime'], now + total + 120):
raise PermissionError('Native policy must cover admission, full lifetime and headroom')
return manifest, grant, policy
class GuardBudget:
"""RootXY cap/deadline facade for reviewed ownership and stdout transport."""
def __init__(self, root, start, mode, monitor=None):
self.root, self.start, self.mode, self.monitor = Path(root), start, mode, monitor
self.boot = b.boot_id()
self.deadline = start + (600 if mode == 'prepare' else 7200)
self.reservation = {'deadline_boottime': self.deadline}
self.interrupted = False
self.external_growth = 0
def check(self, pending=0, *, cleanup=False):
if (self.interrupted and not cleanup or b.boot_id() != self.boot
or not self.start <= b.boottime() < self.deadline - (0 if cleanup else 120)):
raise RuntimeError('RootXY cancelled/deadline/boot mismatch')
if self.monitor is not None:
self.monitor.check()
self.external_growth = sum(max(0, size - self.monitor.baseline[path])
for path, size in self.monitor.latest.items()
if not Path(path).is_relative_to(self.root))
limit = CAP - (0 if cleanup else HEADROOM + STDOUT)
if b.artifact_size(self.root) + self.external_growth + pending > limit:
raise RuntimeError('RootXY aggregate new-owned/native storage cutoff')
def write(self, path, value, *, cleanup=False):
path = Path(path)
if path.parent != path.parent.resolve() or not path.is_relative_to(self.root) or path.is_symlink():
raise ValueError('Unsafe RootXY publication path')
data = canonical(value)
if len(data) > 16 << 20:
raise RuntimeError('Bounded RootXY publication exceeded')
self.check(len(data), cleanup=cleanup)
temporary = path.with_name(path.name + '.pending')
b.durable_write(temporary, data)
os.replace(temporary, path)
b.sync_directory(path.parent)
def desktop_identity(pid, *, hash_executable=True):
"""Pin a pre-existing graphics client, never infer permission from its name."""
if type(pid) is not int or pid <= 0:
raise ValueError('Invalid desktop PID')
proc = Path('/proc') / str(pid)
birth = int((proc / 'stat').read_text().rsplit(') ', 1)[1].split()[19])
uid = proc.stat().st_uid
command = (proc / 'cmdline').read_bytes()
scope = 'proc-exe'
try:
executable = (proc / 'exe').resolve(strict=True)
except PermissionError:
# Ubuntu's root-owned display server hides /proc/PID/exe from this user.
# Explicit, narrowly scoped public-metadata identity, NOT executable proof.
# Still requires a parent-pinned PID/birth/UID/argv and graphics-only NVML.
if (uid != 0 or command.split(b'\0')[0] != b'/usr/lib/xorg/Xorg'
or (proc / 'comm').read_text().strip() != 'Xorg'):
raise
executable = Path('/usr/lib/xorg/Xorg').resolve(strict=True)
scope = 'root-Xorg-public-metadata; actual-executable-unverified'
info = executable.stat()
result = {'pid': pid, 'starttime': birth, 'uid': uid,
'command_sha256': hashlib.sha256(command).hexdigest(),
'executable_identity_scope': scope, 'executable': str(executable),
'device': info.st_dev, 'inode': info.st_ino, 'size': info.st_size,
'mtime_ns': info.st_mtime_ns}
if hash_executable:
result['sha256'] = b.digest(executable)
if (birth != int((proc / 'stat').read_text().rsplit(') ', 1)[1].split()[19])
or command != (proc / 'cmdline').read_bytes() or uid != proc.stat().st_uid
or scope == 'proc-exe' and executable != (proc / 'exe').resolve(strict=True)):
raise RuntimeError('Desktop process changed during identity capture')
return result
class GPUOwnership:
"""Monitor owned GPU work plus explicitly approved existing desktop graphics.
Unknown/new graphics, foreign compute (even an approved desktop PID), changed
process identity or inaccessible NVML fail closed. Desktop clients are NEVER
cleanup-owned. This is shared display monitoring, not GPU partitioning.
"""
class Process(ctypes.Structure):
_fields_ = [('pid', ctypes.c_uint), ('usedGpuMemory', ctypes.c_ulonglong),
('gpuInstanceId', ctypes.c_uint), ('computeInstanceId', ctypes.c_uint)]
def __init__(self, identity):
if (not isinstance(identity, dict) or set(identity) != {'library', 'sha256', 'uuid', 'index', 'desktop_clients'}
or identity['index'] != 0 or type(identity['index']) is not int):
raise ValueError('Exact GPU0 NVML identity required')
path = Path(identity['library'])
if not path.is_absolute() or path != path.resolve() or b.digest(path) != identity['sha256']:
raise ValueError('NVML library identity mismatch')
self.identity = identity
self.lib = ctypes.CDLL(str(path))
self.call('nvmlInit_v2')
self.handle = ctypes.c_void_p()
self.call('nvmlDeviceGetHandleByIndex_v2', ctypes.c_uint(0), ctypes.byref(self.handle))
uuid = ctypes.create_string_buffer(96)
self.call('nvmlDeviceGetUUID', self.handle, uuid, ctypes.c_uint(len(uuid)))
if uuid.value.decode() != identity['uuid']:
raise RuntimeError('GPU UUID mismatch')
clients = identity['desktop_clients']
if (not isinstance(clients, list) or len(clients) > 64
or any(not isinstance(x, dict) or type(x.get('pid')) is not int for x in clients)
or [x['pid'] for x in clients] != sorted({x['pid'] for x in clients})):
raise ValueError('Exact finite desktop client identities required')
self.desktop = {x['pid']: x for x in clients}
actual = self.clients()
if actual['compute'] or actual['graphics'] != set(self.desktop):
raise RuntimeError('GPU admission compute/approved-desktop mismatch')
for pid, expected in self.desktop.items():
if desktop_identity(pid) != expected:
raise RuntimeError('GPU desktop admission identity mismatch')
def call(self, name, *args):
code = getattr(self.lib, name)(*args)
if code != 0:
raise RuntimeError('NVML query failed: ' + name + ':' + str(code))
def clients(self):
result = {}
for kind, name in (('compute', 'nvmlDeviceGetComputeRunningProcesses_v2'),
('graphics', 'nvmlDeviceGetGraphicsRunningProcesses_v2')):
count = ctypes.c_uint(1024)
processes = (self.Process * 1024)()
self.call(name, self.handle, ctypes.byref(count), processes)
if count.value > 1024:
raise RuntimeError('GPU ownership list overflow')
result[kind] = {processes[i].pid for i in range(count.value)}
return result
def pids(self):
clients = self.clients()
return clients['compute'] | clients['graphics']
def check(self, owned, *, final=False):
if b.digest(self.identity['library']) != self.identity['sha256']:
raise RuntimeError('NVML drift')
clients = self.clients()
active = set(owned.discover())
graphics = clients['graphics']
desktop = graphics & set(self.desktop)
for pid in desktop:
expected = self.desktop[pid]
if not final:
expected = {k: v for k, v in expected.items() if k != 'sha256'}
if desktop_identity(pid, hash_executable=final) != expected:
raise RuntimeError('GPU desktop process identity changed')
if (clients['compute'] - active or graphics - active - desktop
or clients['compute'] & set(self.desktop)
or final and (clients['compute'] or graphics - desktop)):
raise RuntimeError('GPU ownership lost or residual task/foreign GPU client')
return {'uuid': self.identity['uuid'], 'compute_pids': sorted(clients['compute']),
'approved_desktop_graphics_pids': sorted(desktop),
'task_graphics_pids': sorted(graphics - desktop),
'clean': not clients['compute'] and not (graphics - desktop),
'scope': 'owned-compute-and-graphics.plus-pinned-desktop-graphics-only'}
def worker_payload(fd):
"""RootXY entrypoint-specific authenticated single-use inherited FD."""
with socket.socket(fileno=fd) as channel:
channel.settimeout(15)
pid, uid, _ = struct.unpack('3i', channel.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
command = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0')
entry = command[1:] # only harmless interpreter flags may precede the actual entry script
while entry and entry[0] in (b'-u', b'-B', b'-s', b'-E', b'-I'):
entry = entry[1:]
script_matches = bool(entry and Path(entry[0].decode()).is_absolute()
and Path(entry[0].decode()) == ROOT / SCRIPT)
if (not script_matches or Path(f'/proc/{pid}/exe').resolve() != Path('/usr/bin/python3.12')
or not owner.ancestor(pid, os.getpid()) or uid != os.getuid()
or not (b'--run' in entry or b'--prepare' in entry)):
raise PermissionError('Authenticated RootXY supervisor required')
channel.sendall(str(os.getpid()).encode() + b'\n')
with channel.makefile('rb') as stream:
data = stream.read(1 << 20)
if len(data) >= 1 << 20:
raise ValueError('Oversized RootXY handoff')
payload = json.loads(data)
if payload['launch_identity'] != launch_identity():
raise ValueError('RootXY launch drift')
return payload
def validate_completion(root, result, manifest, start):
data = read(root / 'budget.json')
expected = {'reserved_transitions': 1048576, 'reservations': 256, 'pending': None,
'updates': {'92511': 256}, 'validation_reserved_transitions': 806400,
'validation_attempted_control_calls': 5400, 'validation_completed_control_calls': 5400,
'validation_evaluations_started': ['analytic_zero', 'frozen_u256', 'u0064', 'u0128',
'u0192', 'u0256']}
if any(data.get(k) != v or result.get('budget', {}).get(k) != v for k, v in expected.items()):
raise ValueError('Incomplete optimization/validation debit evidence')
if (data['clock']['start_s'] != start or data['clock']['deadline_s'] != start + 7200
or data['clock']['boot_id'] != b.boot_id()):
raise ValueError('Worker changed supervisor lifetime')
physics = result['physics']
attempted, completed, init, rollout, startup = [physics[k] for k in (
'attempted_shared_calls', 'completed_shared_calls', 'initialization_attempted_shared_calls',
'rollout_attempted_shared_calls', 'startup_completed_without_task_audit')]
if (any(type(x) is not int for x in (attempted, completed, init, rollout, startup))
or not 0 <= startup <= init <= 1000 or rollout != 67960
or attempted != completed or completed != init + rollout or completed > 73460):
raise ValueError('Actual shared physics completion/cap evidence mismatch')
safety = physics['safety']
gates = read(ROOT / manifest['training']['safety_config'])['gates']
values = [safety[k] for k in ('max_effort_velocity_ratio', 'max_joint_excess_rad', 'max_guide_vertical_N')]
if (safety['physics_samples'] != completed - startup or safety['finite'] is not True
or any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values)
or values[0] > gates['normalized_effort_velocity_max']
or values[1] > gates['joint_limit_excess_rad_max'] or values[2] != 0):
raise ValueError('Missing/failing global physical safety evidence')
for label in ('analytic_zero', 'frozen_u256'):
rows = read(root / ('validation_' + label + '.json'))['rows']
if (len(rows) != 128 or [r.get('env') for r in rows] != list(range(128))
or any(r.get('success') is not True or r.get('functional_audit', {}).get('sim_functional_passed')
is not True for r in rows)):
raise ValueError('Missing original baseline functional128/128 evidence')
def commit_candidate(root, result, manifest, budget):
"""Independent validation and sole-authority last atomic commit; no worker lock."""
# Import CPU protocol only after worker exit/cleanup; never initialize CUDA.
selection_path = ROOT / owner.PACKAGE / 'legs/contact_cycle_sim/rootxy_selection_v2.py'
spec = importlib.util.spec_from_file_location('_rootxy_selection_cpu', selection_path)
selection = importlib.util.module_from_spec(spec)
spec.loader.exec_module(selection)
if (result.get('status') != 'completed_provisional' or result.get('training_complete') is not True
or result.get('accepted_skill') is not False or result.get('checkpoints_selectable') is not False
or result.get('postclose_identity_verified') is not True):
raise ValueError('No successful provisional worker result')
ranked = []
for update in (64, 128, 192, 256):
document = read(root / f'validation_u{update:04d}.json')
rows = document['rows']
# Recheck original functional audit, not just worker's candidate label.
if any(r.get('functional_audit', {}).get('sim_functional_passed') is not True for r in rows):
raise ValueError('Missing original functional128/128 evidence')
key = selection.candidate_key(rows, update)
if key is not None:
ranked.append((key, update))
candidate = result.get('candidate')
if not ranked:
if candidate is not None or result.get('outcome') != 'no_candidate':
raise ValueError('Ineligible worker candidate')
return None
key, update = min(ranked)
if (not isinstance(candidate, dict) or set(candidate) != {'path', 'sha256', 'update', 'selection_key'}
or candidate['update'] != update or candidate['path'] != f'checkpoint_u{update:04d}.pt'
or candidate['selection_key'] != list(key) or b.digest(root / candidate['path']) != candidate['sha256']):
raise ValueError('Candidate selection/hash mismatch')
lock = {'format': 'skillfactory.rootxy-supervisor-candidate.v2', 'candidate': candidate,
'manifest_file_sha256': b.digest(root / 'manifest.json'), 'accepted_skill': False,
'authority': 'waited_exit_postexit_identity_clean_owned_GPU_cleanup_original_eligibility',
'downstream_requires_separate_grant': True}
budget.write(root / 'locked_candidate.json', lock, cleanup=True)
return lock
def supervise(args):
start = b.boottime() # includes admission/startup; never reset in worker
manifest, grant, policy = admission(args, start)
mode = 'prepare' if args.prepare else 'run'
if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'):
raise RuntimeError('pidfd supervisor required')
owned = owner.OwnedProcesses()
gpu = GPUOwnership(grant['gpu_identity'])
environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
CUDA_VISIBLE_DEVICES=grant['gpu_identity']['uuid'])
owner.launch_command([], environment)
if mode == 'prepare':
b.durable_directory(args.output)
lock = (args.output / (mode + '-claim.lock')).open('xb')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
budget = GuardBudget(args.output, start, mode)
monitor = child = capture = channels = None
errors, handlers = [], {}
clean = waited = False
result = None
filesystem = None
postexit = None
def cleanup_pump():
if capture is not None:
capture.pump()
if monitor is not None:
try:
monitor.check()
except BaseException as error:
if not any(e.startswith('native_cleanup:') for e in errors):
errors.append('native_cleanup:' + str(error)[-2048:])
try:
monitor = owner.NativeMonitor(policy, SimpleNamespace(boottime=b.boottime, STORAGE_SCOPE=STORAGE_SCOPE))
budget.monitor = monitor
handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True))
for s in (signal.SIGTERM, signal.SIGINT)}
budget.write(args.output / (mode + '-reservation.json'), {
'profile': PROFILE, 'mode': mode, 'start_boottime': start, 'deadline_boottime': budget.deadline,
'boot_id': budget.boot, 'grant': grant,
'reserved_optimization_environment_transitions': 0 if mode == 'prepare' else 1048576,
'reserved_validation_environment_transitions': 0 if mode == 'prepare' else 806400,
'reserved_rollout_shared_calls': 0 if mode == 'prepare' else 72460,
'initialization_shared_calls_cap': 0 if mode == 'prepare' else 1000,
'no_refund': True, 'accepted_skill': False})
budget.write(args.output / (mode + '-manifest.json'), manifest)
if mode == 'prepare':
for relative, digest in manifest['sources'].items():
destination = args.output / 'source_snapshot' / relative
destination.parent.mkdir(parents=True, exist_ok=True)
data = (ROOT / relative).read_bytes()
budget.check(len(data))
b.durable_write(destination, data)
original_manifest = Path(args.manifest).read_bytes()
budget.check(len(original_manifest))
b.durable_write(args.output / 'manifest.json', original_manifest)
filesystem = owner.bound_filesystem_inputs(original_manifest)
else:
filesystem = owner.bound_filesystem_inputs((args.output / 'training-binding.json').read_bytes())
owner.verify_filesystem_inputs(filesystem)
budget.write(args.output / (mode + '-grant.json'), grant)
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
raise RuntimeError('Subreaper setup failed')
native = args.output / 'native'
if native != native.resolve():
raise ValueError('Native redirect symlink forbidden')
native.mkdir(exist_ok=True)
for directory in redirects(args.output, mode).values():
Path(directory).mkdir(parents=True, exist_ok=False)
environment.update(redirects(args.output, mode))
channels = socket.socketpair()
parent, child_channel = channels
child_channel.set_inheritable(True)
budget.check()
child = subprocess.Popen(owner.launch_command(
[str(ROOT / SCRIPT), '--_worker-fd', str(child_channel.fileno())], environment),
cwd=ROOT, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
start_new_session=True, pass_fds=(child_channel.fileno(),))
try:
owned.attach(child)
except BaseException:
owned.unknown = True
raise
child_channel.close()
capture = owner.StdoutCapture(child.stdout, args.output / (mode + '-stdout.log'), b)
budget.write(args.output / 'monitor-heartbeat.json', {
'boot_id': budget.boot, 'boottime': b.boottime(), 'external_growth': budget.external_growth})
owner.exchange_worker(child, parent, {
'mode': mode, 'output': str(args.output), 'start': start, 'boot_id': budget.boot,
'manifest_sha256': args.trusted_manifest_sha256, 'grant': grant,
'launch_identity': launch_identity()}, budget, capture, owned, monitor, b)
next_heartbeat = 0
while child.poll() is None or not capture.eof:
capture.check()
budget.check()
gpu.check(owned)
if b.boottime() >= next_heartbeat:
budget.write(args.output / 'monitor-heartbeat.json', {
'boot_id': budget.boot, 'boottime': b.boottime(), 'external_growth': budget.external_growth})
next_heartbeat = b.boottime() + .5
time.sleep(.01)
child.wait(timeout=max(.01, budget.deadline - b.boottime()))
waited = True
result = read(args.output / (mode + '-worker-result.json'))
if child.returncode != 0:
raise RuntimeError('Worker nonzero exit')
except BaseException as error:
errors.append(str(error)[-8192:])
finally:
try:
if channels:
for channel in channels:
channel.close()
if child is not None:
if capture is None and child.stdout is not None:
child.stdout.close()
try:
clean = owner.cleanup_group(child, budget.deadline, b, owned, cleanup_pump)
child.wait(timeout=max(.01, budget.deadline - b.boottime()))
waited = True
except BaseException as error:
errors.append('cleanup:' + str(error)[-2048:])
if capture is not None:
try:
capture.close()
if capture.discarded or capture.error or not capture.eof:
raise RuntimeError('Incomplete/overflowed stdout')
except BaseException as error:
errors.append(str(error)[-2048:])
try:
if not clean or not waited or child is None or child.returncode != 0 or owned.unknown:
raise RuntimeError('No clean independently waited worker exit')
gpu.check(owned, final=True)
verify_sources(manifest)
if filesystem is None:
raise RuntimeError('Missing prelaunch filesystem identity')
postexit = owner.verify_filesystem_inputs(filesystem)
binding_files = owner.bound_filesystem_inputs((args.output / 'training-binding.json').read_bytes())
owner.verify_filesystem_inputs(binding_files)
if result is None or result.get('postclose_identity_verified') is not True:
raise RuntimeError('Missing postclose same-object verification')
if mode == 'prepare':
if result.get('status') != 'prepared_config_only' or result.get('physics_calls') != 0:
raise RuntimeError('PREP incomplete/config-only violation')
else:
if result.get('status') != 'completed_provisional' or result.get('training_complete') is not True:
raise RuntimeError('Training incomplete')
validate_completion(args.output, result, manifest, start)
budget.check(cleanup=True)
monitor.check(force=True)
except BaseException as error:
errors.append(str(error)[-4096:])
summary = {'success': not errors, 'mode': mode, 'cleanup_clean': clean, 'waited_exit': waited,
'returncode': None if child is None else child.returncode, 'errors': errors,
'accepted_skill': False, 'postexit': postexit, 'owned': sorted(owned.known),
'storage': None if monitor is None else monitor.report()}
budget.write(args.output / (mode + '-supervisor.json'), summary, cleanup=True)
if not errors and mode == 'prepare':
prepared = read(args.output / 'config-preparation-provisional.json')
prepared.update(supervisor_sha256=b.digest(args.output / 'prepare-supervisor.json'),
worker_result_sha256=b.digest(args.output / 'prepare-worker-result.json'),
postexit_verified=True)
try:
budget.write(args.output / 'preparation.json', prepared, cleanup=True)
except BaseException:
(args.output / 'preparation.json').unlink(missing_ok=True)
raise
if not errors and mode == 'run':
# Last operation capable of committing success. Failure leaves no candidate.
try:
commit_candidate(args.output, result, manifest, budget)
except BaseException as error:
(args.output / 'locked_candidate.json').unlink(missing_ok=True)
summary.update(success=False, errors=[*errors, 'candidate_commit:' + str(error)[-2048:]])
budget.write(args.output / (mode + '-supervisor.json'), summary, cleanup=True)
raise
finally:
for sig, handler in handlers.items():
signal.signal(sig, handler)
lock.close()
return 0 if not errors else 1
def preimport_binding_runtime(binding):
"""Finite binding-required imports, before config/inventory; no optional skips."""
for name in dict.fromkeys((*binding.REQUIRED_RUNTIME, *binding.ISAAC_PINS)):
importlib.import_module(name)
class RuntimeAPI:
"""Actual Kit recipe; callable only from authenticated worker handoff."""
def __init__(self, payload, budget, guard):
self.payload, self.budget, self.guard = payload, budget, guard
self.output = Path(payload['output'])
self.mode = payload['mode']
self.launcher = self.env = self.cfg = self.ledger = None
self.started = False
self.physics_hooks = None
self.manifest = read(self.output / 'manifest.json', payload['manifest_sha256'])
self.expected = None
def start(self):
from isaaclab.app import AppLauncher
app_source = self.payload['launch_identity']['app_launcher']
if (AppLauncher.__module__ != 'isaaclab.app.app_launcher'
or Path(sys.modules[AppLauncher.__module__].__file__).resolve() != Path(app_source['path'])
or b.digest(app_source['path']) != app_source['sha256']):
raise ValueError('Actual installed AppLauncher identity mismatch')
# Retain launcher even if initialization fails after creating its app.
self.launcher = AppLauncher.__new__(AppLauncher)
AppLauncher.__init__(self.launcher, headless=True, device='cuda:0', fast_shutdown=True,
kit_args=f'--/log/file={self.output / "native" / self.mode / "kit.log"}')
self.started = True
for name in ('rootxy_training_v2', 'rootxy_runner_v2', 'rootxy_binding_v2', 'rootxy_runtime_v2'):
setattr(self, name, importlib.import_module(owner.LEG + name))
self.p = self.rootxy_training_v2
self.binding = self.rootxy_binding_v2
eb = importlib.import_module(owner.LEG + 'evaluation_binding')
self.eb = eb
for name in self.binding.ROLES:
if name != '__main__':
importlib.import_module(name)
# Do not depend on an evaluation recorder to trigger lazy backends.
preimport_binding_runtime(eb)
# Match the finite installed constructor preimports, never a whole interpreter scan.
owner.preimport_replicator()
owner.preimport_constructor_modules(eb)
self.constructor_export, self.constructor_metadata = owner.physx_constructor_export(eb, resolve=True)
self.cfg_class = importlib.import_module(owner.TASK + 'rootxy_cfg_v2').RootXYContactCycleEnvCfg
self.environment_class = importlib.import_module(owner.TASK + 'env').SimContactCycleEnv
def configure(self):
self.cfg = self.cfg_class()
self.cfg.scene.num_envs = 128
self.cfg.seed = 92511
self.cfg.sim.device = 'cuda:0'
# This helper only verifies installed stepping and explicitly forwards
# fixed constructor video config fields; no evaluation environment created.
owner.preforward_video_recorder(self.cfg, importlib.import_module(owner.TASK + 'evaluation_env'))
owner.preimport_config_references(self.cfg, self.eb)
self.binding.normalize_config(self.cfg) # exact128-env PREconstruction only
import torch
torch.cuda.init()
torch.cuda.get_rng_state(torch.device('cuda:0'))
if self.mode == 'prepare':
self.inventory = self.binding.propose_inventory(ROOT, self.cfg, owner.proposed_native_layouts)
self.expected = self.binding.snapshot(ROOT, self.output, self.cfg, self.inventory, self.manifest)
else:
self.expected = read(self.output / 'training-binding.json', self.payload['grant']['binding_sha256'])
self.inventory = self.expected['dependency_inventory']
if self.mode == 'run':
outer = self
class Ledger(self.p.RootXYBudgetLedger):
def guard(self, reserve_bytes=0):
outer.guard()
outer.budget.check(reserve_bytes)
return super().guard(reserve_bytes=reserve_bytes)
self.ledger = Ledger(self.output, self.manifest, launch_start_s=self.payload['start'])
self.ledger.__enter__()
@contextmanager
def physics(self):
from isaaclab.sim import SimulationContext
with self.rootxy_runtime_v2.RootXYPhysicsHooks(
SimulationContext, self.environment_class, self.guard,
lambda value: self.budget.write(self.output / 'physics-counts.json', value),
preparation=self.mode == 'prepare') as hooks:
# Config construction/normalization itself is under zero-physics PREP hooks.
self.configure()
yield hooks
def verify(self):
self.budget.check(cleanup=True)
verify_sources(self.manifest)
if self.cfg is None or self.expected is None:
raise RuntimeError('Startup did not establish actual config identity')
target, metadata = owner.physx_constructor_export(self.eb, expected=self.constructor_export)
if metadata != self.constructor_metadata:
raise ValueError('Constructor export drift')
self.binding.verify(self.expected, root=ROOT, output=self.output, cfg=self.cfg,
inventory=self.inventory, manifest=self.manifest)
if self.env is not None and getattr(self.env, 'cfg', None) is not self.cfg:
raise ValueError('Constructor substituted actual config object')
def prepare(self):
self.budget.write(self.output / 'training-binding.json', self.expected)
result = {'format': PREPARATION_FORMAT, 'status': 'prepared_config_only',
'manifest_sha256': self.payload['manifest_sha256'],
'binding_sha256': b.digest(self.output / 'training-binding.json'),
'artifact_root': str(self.output), 'physics_calls': 0, 'num_envs': 128,
'runtime_qualified': False, 'p2_decision': 'pending', 'accepted_skill': False,
'recipe': {'device': 'cuda:0', 'train_seed': 92511, 'development_seed': 93511,
'environment_constructed': False, 'inference_executed': False,
'profile': PROFILE, 'hooks': self.binding.HOOKS},
'ready_gaps': ['parent review of actual config-only binding',
'separate approved P2 decision and fresh run grant',
'immediate postconstructor actual-used identity and reset safety qualification']}
self.budget.write(self.output / 'config-preparation-provisional.json', result)
return result
def construct(self):
self.guard()
self.env = self.environment_class.__new__(self.environment_class)
self.environment_class.__init__(self.env, cfg=self.cfg)
def check_actual(self):
importlib.import_module(owner.TASK + 'rootxy_cfg_v2').verify_actual_config(self.env)
if self.env._physics_handles_decimation:
raise RuntimeError('Require scalar 500Hz physics stepping')
def safety(self):
from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit
gates = read(ROOT / self.p.PILOT['safety_config'])['gates']
return PhysicalSafetyAudit(self.env, gates)
def learn(self):
import torch
from audit_contact_cycle_functional import evaluate
trace_path = ROOT / self.p.replay.CANDIDATE
trace_path = trace_path.parent / 'validation_u0256_trace.pt'
if b.digest(trace_path) != self.manifest['parity_trace']['sha256']:
raise ValueError('Saved CPU parity trace drift')
trace = torch.load(trace_path, map_location='cpu', weights_only=True)
observations = next(iter(trace.values()))['actor_input'][::100, 0].contiguous()
del trace
def guard():
self.guard()
self.physics_hooks.safety_boundary()
def before_control(kind):
if kind not in ('optimization', 'validation'):
raise ValueError('Unknown control budget kind')
guard()
self.physics_hooks.mode = 'rollout'
def after_control(kind):
self.physics_hooks.mode = 'initialization'
guard()
hooks = self.rootxy_runner_v2.RuntimeHooks(
admit=lambda manifest: {'manifest_sha256': self.p.digest(manifest), 'p2_review_approved': True,
'sole_cleanup_owner': True, 'native_monitor_active': True},
guard=guard, verify_identity=self.verify, before_control=before_control, after_control=after_control,
progress=lambda value: guard(), finish=lambda value: guard())
return self.rootxy_runner_v2.execute(self.env, ROOT, self.output, self.manifest, self.ledger,
observations, hooks, evaluate)
def close_env(self):
if self.env is not None:
self.env.close()
def close_app(self, code):
error = None
try:
if self.ledger is not None:
self.ledger.__exit__()
except BaseException as caught:
error = caught
app = None if self.launcher is None else getattr(self.launcher, '_app', None)
if app is not None:
# Fast shutdown may exit instead of returning: never pass zero after
# another teardown failure and rely on later Python unwinding.
app.close(exit_code=1 if error is not None else code)
if error is not None:
raise error
def run_worker(fd):
payload = worker_payload(fd) # no unauthenticated dictionary entrypoint
if payload['mode'] not in ('prepare', 'run') or payload['boot_id'] != b.boot_id():
raise PermissionError('Invalid authenticated RootXY handoff')
root = checked_root(payload['output'])
reservation = read(root / (payload['mode'] + '-reservation.json'))
if (reservation['grant'] != payload['grant'] or reservation['start_boottime'] != payload['start']
or reservation['boot_id'] != payload['boot_id']):
raise PermissionError('RootXY reservation mismatch')
budget = GuardBudget(root, payload['start'], payload['mode'])
def guard():
heartbeat = read(root / 'monitor-heartbeat.json')
if (heartbeat['boot_id'] != b.boot_id()
or not 0 <= b.boottime() - heartbeat['boottime'] <= 5):
raise RuntimeError('Native monitor heartbeat missing/stale')
budget.external_growth = heartbeat['external_growth']
budget.check()
handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True))
for s in (signal.SIGTERM, signal.SIGINT)}
# Pure hook module is loaded under a private identity before Kit, not a fake
# SkillFactory package. Actual runtime imports after AppLauncher use real paths.
spec = importlib.util.spec_from_file_location('_rootxy_lifecycle_cpu', ROOT / owner.PACKAGE /
'legs/contact_cycle_sim/rootxy_runtime_v2.py')
runtime = importlib.util.module_from_spec(spec)
spec.loader.exec_module(runtime)
api = RuntimeAPI(payload, budget, guard)
verify_sources(api.manifest)
try:
runtime.worker_lifecycle(api, prepare=payload['mode'] == 'prepare', guard=guard, verify=api.verify,
publish=lambda value: budget.write(
root / (payload['mode'] + '-worker-result.json'), value, cleanup=True))
return 0
finally:
for sig, handler in handlers.items():
signal.signal(sig, handler)