"""Dedicated RootXY supervisor adapter. Importing this module never starts Kit. Ownership/native monitoring/launch transport reuse reviewed evaluator helpers; no evaluation grant, protocol, ledger, checkpoint or candidate authority is reused. """ import argparse import ctypes from contextlib import contextmanager import fcntl import hashlib import importlib import json import math import os import signal import socket import struct import subprocess import sys import time from pathlib import Path from types import SimpleNamespace import evaluate_cycle_sim_residual as owner ROOT = owner.ROOT SCRIPT = 'scripts/train_cycle_sim_rootxy_v2.py' PROFILE = 'rootxy-v2-7200s-1GiB-73460calls' TASK_ID = 'SkillFactory-Go2FL-ContactCycleSimRootXY-v2' PREREG_FORMAT = 'skillfactory.contact-cycle-sim-rootxy-preregistration.v2' GRANT_FORMAT = 'skillfactory.rootxy-parent-grant.v2' POLICY_FORMAT = 'skillfactory.rootxy-native-policy.v2' PREPARATION_FORMAT = 'skillfactory.rootxy-config-preparation.v2' STORAGE_SCOPE = 'rootxy-new-owned.native-monitored-stop.v2' DECISION_FORMAT = 'skillfactory.rootxy-p2-decision.v2' CAP = 1 << 30 HEADROOM = 16 << 20 STDOUT = 1 << 20 NATIVE_LIFETIME = 10800 b = owner.cpu_module('evaluation_budget') def canonical(value): return b.encode(value) def sha(value): return hashlib.sha256(canonical(value)).hexdigest() def read(path, expected=None): path = Path(path) if not path.is_file() or path.is_symlink() or path.stat().st_size > 32 << 20: raise ValueError('Missing/unsafe/oversized RootXY input: ' + str(path)) data = path.read_bytes() if expected is not None: owner.read_trusted(path, expected) return json.loads(data) def checked_root(path): root = Path(path) if (not root.is_absolute() or root != root.resolve() or root.is_relative_to(ROOT) and not root.is_relative_to(ROOT / 'logs/contact_cycle/rootxy_v2') or root.is_relative_to(ROOT / 'releases') or root.is_relative_to(ROOT / 'logs/contact_cycle/sim_residual_training_v1')): raise ValueError('Canonical new isolated RootXY root required') return root def verify_sources(manifest): if (manifest.get('format') != PREREG_FORMAT or manifest.get('task_id') != TASK_ID or manifest.get('launch_integration_complete') is not True or manifest.get('accepted_skill') is not False): raise ValueError('New RootXY preregistration required') for relative, expected in manifest['sources'].items(): path = ROOT / relative if (Path(relative).is_absolute() or '..' in Path(relative).parts or path != path.resolve() or b.digest(path) != expected): raise ValueError('RootXY source drift: ' + relative) cfg = manifest['training'] fixed = {'seeds': [92511], 'validation_seed': 93511, 'num_envs': 128, 'max_total_wall_s': 7200, 'cleanup_reserve_s': 120, 'max_artifact_bytes': CAP, 'max_validation_transitions': 806400, 'validation_updates': [64, 128, 192, 256]} if any(cfg.get(k) != v for k, v in fixed.items()): raise ValueError('RootXY resource contract changed') def launch_identity(): app_path = Path('/home/cen/IsaacLab/source/isaaclab/isaaclab/app/app_launcher.py') return {'app_launcher': {'path': str(app_path), 'sha256': b.digest(app_path)}, 'format': 'skillfactory.rootxy-launch-identity.v2', 'transport': owner.launch_identity(), 'profile': PROFILE, 'task_id': TASK_ID, 'entrypoint': {'path': str(ROOT / SCRIPT), 'sha256': b.digest(ROOT / SCRIPT)}, 'adapter': {'path': str(Path(__file__).resolve()), 'sha256': b.digest(__file__)}} def redirects(root, mode): return owner.native_redirects(root / 'native' / mode, preparation=True) def policy_check(args, now): policy = read(args.storage_policy, args.trusted_storage_policy_sha256) root = args.output fields = {'format', 'profile', 'artifact_root', 'limit_bytes', 'native_paths', 'shared_native_paths', 'task_owned_cache_paths', 'free_space_paths', 'free_space_mount_identities', 'mount_identity', 'check_interval_seconds', 'max_native_growth_bytes', 'min_free_bytes', 'coverage_gaps', 'native_hard_cap', 'boot_id', 'issued_boottime', 'not_after_boottime', 'lifetime_seconds'} duration = policy.get('lifetime_seconds') if (set(policy) != fields or policy['format'] != POLICY_FORMAT or policy['profile'] != PROFILE or policy['artifact_root'] != str(root) or policy['limit_bytes'] != CAP or policy['native_paths'] != owner.native_paths(root) or policy['shared_native_paths'] != owner.shared_native_paths() or policy['task_owned_cache_paths'] != sorted([*redirects(root, 'prepare').values(), *redirects(root, 'run').values()]) or policy['free_space_paths'] != owner.free_space_paths(root) or policy['free_space_mount_identities'] != { p: owner.mount_identity(Path(p)) for p in owner.free_space_paths(root)} or policy['mount_identity'] != owner.mount_identity(root) or policy['coverage_gaps'] != list(owner.NATIVE_GAPS) or policy['native_hard_cap'] is not False or policy['boot_id'] != b.boot_id() or type(duration) is not int or not 7200 <= duration <= 10800 or type(policy['issued_boottime']) not in (int, float) or not 0 <= policy['issued_boottime'] <= now or policy['not_after_boottime'] != policy['issued_boottime'] + duration or type(policy['check_interval_seconds']) not in (int, float) or not 0 < policy['check_interval_seconds'] <= 1 or type(policy['max_native_growth_bytes']) is not int or not 0 < policy['max_native_growth_bytes'] <= CAP or type(policy['min_free_bytes']) is not int or policy['min_free_bytes'] < 2 * CAP): raise PermissionError('Exact RootXY native MONITOR policy required (not quota)') return policy def admission(args, now): args.output = checked_root(args.output) mode = 'prepare' if args.prepare else 'run' required = ('manifest', 'trusted_manifest_sha256', 'authorization', 'trusted_authorization_sha256', 'storage_policy', 'trusted_storage_policy_sha256') if any(getattr(args, k, None) is None for k in required): raise PermissionError('Fresh exact RootXY manifest/native-policy/parent grant required') manifest = read(args.manifest, args.trusted_manifest_sha256) verify_sources(manifest) grant = read(args.authorization, args.trusted_authorization_sha256) fields = {'format', 'profile', 'task_id', 'mode', 'artifact_root', 'manifest_sha256', 'storage_policy_sha256', 'launch_identity_sha256', 'boot_id', 'not_after_boottime', 'parent_review', 'preparation_sha256', 'binding_sha256', 'p2_decision_sha256', 'gpu_identity'} if (set(grant) != fields or grant['format'] != GRANT_FORMAT or grant['profile'] != PROFILE or grant['task_id'] != TASK_ID or grant['mode'] != mode or grant['artifact_root'] != str(args.output) or grant['manifest_sha256'] != args.trusted_manifest_sha256 or grant['storage_policy_sha256'] != args.trusted_storage_policy_sha256 or grant['launch_identity_sha256'] != sha(launch_identity()) or grant['boot_id'] != b.boot_id() or type(grant['not_after_boottime']) not in (int, float) or not now < grant['not_after_boottime'] <= now + 3600 or type(grant['parent_review']) is not str or not grant['parent_review'].strip()): raise PermissionError('RootXY parent grant identity/expiry mismatch') if mode == 'prepare': if any(grant[k] is not None for k in ('preparation_sha256', 'binding_sha256', 'p2_decision_sha256')): raise PermissionError('PREP has pending P2/config identity only, not runtime approval') if args.output.exists(): raise FileExistsError('PREP requires a NEW root; no retry') else: for field in ('preparation_sha256', 'binding_sha256', 'p2_decision_sha256'): value = grant[field] if (type(value) is not str or len(value) != 64 or any(c not in '0123456789abcdef' for c in value)): raise PermissionError('RUN requires an exact independently approved ' + field) if any((args.output / name).exists() for name in ( 'run-claim.lock', 'launch_claim.json', 'budget.json', 'budget.lock', 'locked_candidate.json', 'run-worker-result.json', 'run-supervisor.json')): raise FileExistsError('Fresh-only RootXY RUN; no prior trial or candidate artifacts') preparation = read(args.output / 'preparation.json', grant['preparation_sha256']) binding = read(args.output / 'training-binding.json', grant['binding_sha256']) decision = read(args.p2_decision, grant['p2_decision_sha256']) if (preparation.get('format') != PREPARATION_FORMAT or preparation.get('status') != 'prepared_config_only' or preparation.get('manifest_sha256') != args.trusted_manifest_sha256 or preparation.get('binding_sha256') != grant['binding_sha256'] or preparation.get('artifact_root') != str(args.output) or preparation.get('physics_calls') != 0 or preparation.get('runtime_qualified') is not False or binding.get('format') != 'skillfactory.rootxy-training-binding.v2' or binding.get('artifact_root') != str(args.output) or binding.get('profile') != PROFILE): raise PermissionError('Exact config-only preparation/binding required') prep_supervisor = read(args.output / 'prepare-supervisor.json', preparation['supervisor_sha256']) read(args.output / 'prepare-worker-result.json', preparation['worker_result_sha256']) if (preparation.get('postexit_verified') is not True or prep_supervisor.get('success') is not True or prep_supervisor.get('cleanup_clean') is not True): raise PermissionError('PREP needs independent clean waited-exit completion') if (set(decision) != {'format', 'task_id', 'artifact_root', 'manifest_sha256', 'preparation_sha256', 'binding_sha256', 'approved', 'parent_review', 'p2_evidence'} or decision['format'] != DECISION_FORMAT or decision['task_id'] != TASK_ID or decision['artifact_root'] != str(args.output) or decision['approved'] is not True or decision['manifest_sha256'] != args.trusted_manifest_sha256 or decision['preparation_sha256'] != grant['preparation_sha256'] or decision['binding_sha256'] != grant['binding_sha256'] or not isinstance(decision['parent_review'], str) or not decision['parent_review'].strip() or not isinstance(decision['p2_evidence'], dict) or not decision['p2_evidence']): raise PermissionError('Separate parent-approved P2 evidence/feasibility decision required') for path, digest in decision['p2_evidence'].items(): owner.read_trusted(path, digest) for relative, digest in manifest['sources'].items(): if b.digest(args.output / 'source_snapshot' / relative) != digest: raise ValueError('RootXY preparation snapshot drift') policy = policy_check(args, now) total = 600 if mode == 'prepare' else 7200 if policy['not_after_boottime'] < max(grant['not_after_boottime'], now + total + 120): raise PermissionError('Native policy must cover admission, full lifetime and headroom') return manifest, grant, policy class GuardBudget: """RootXY cap/deadline facade for reviewed ownership and stdout transport.""" def __init__(self, root, start, mode, monitor=None): self.root, self.start, self.mode, self.monitor = Path(root), start, mode, monitor self.boot = b.boot_id() self.deadline = start + (600 if mode == 'prepare' else 7200) self.reservation = {'deadline_boottime': self.deadline} self.interrupted = False self.external_growth = 0 def check(self, pending=0, *, cleanup=False): if (self.interrupted and not cleanup or b.boot_id() != self.boot or not self.start <= b.boottime() < self.deadline - (0 if cleanup else 120)): raise RuntimeError('RootXY cancelled/deadline/boot mismatch') if self.monitor is not None: self.monitor.check() self.external_growth = sum(max(0, size - self.monitor.baseline[path]) for path, size in self.monitor.latest.items() if not Path(path).is_relative_to(self.root)) limit = CAP - (0 if cleanup else HEADROOM + STDOUT) if b.artifact_size(self.root) + self.external_growth + pending > limit: raise RuntimeError('RootXY aggregate new-owned/native storage cutoff') def write(self, path, value, *, cleanup=False): path = Path(path) if path.parent != path.parent.resolve() or not path.is_relative_to(self.root) or path.is_symlink(): raise ValueError('Unsafe RootXY publication path') data = canonical(value) if len(data) > 16 << 20: raise RuntimeError('Bounded RootXY publication exceeded') self.check(len(data), cleanup=cleanup) temporary = path.with_name(path.name + '.pending') b.durable_write(temporary, data) os.replace(temporary, path) b.sync_directory(path.parent) def desktop_identity(pid, *, hash_executable=True): """Pin a pre-existing graphics client, never infer permission from its name.""" if type(pid) is not int or pid <= 0: raise ValueError('Invalid desktop PID') proc = Path('/proc') / str(pid) birth = int((proc / 'stat').read_text().rsplit(') ', 1)[1].split()[19]) uid = proc.stat().st_uid command = (proc / 'cmdline').read_bytes() scope = 'proc-exe' try: executable = (proc / 'exe').resolve(strict=True) except PermissionError: # Ubuntu's root-owned display server hides /proc/PID/exe from this user. # Explicit, narrowly scoped public-metadata identity, NOT executable proof. # Still requires a parent-pinned PID/birth/UID/argv and graphics-only NVML. if (uid != 0 or command.split(b'\0')[0] != b'/usr/lib/xorg/Xorg' or (proc / 'comm').read_text().strip() != 'Xorg'): raise executable = Path('/usr/lib/xorg/Xorg').resolve(strict=True) scope = 'root-Xorg-public-metadata; actual-executable-unverified' info = executable.stat() result = {'pid': pid, 'starttime': birth, 'uid': uid, 'command_sha256': hashlib.sha256(command).hexdigest(), 'executable_identity_scope': scope, 'executable': str(executable), 'device': info.st_dev, 'inode': info.st_ino, 'size': info.st_size, 'mtime_ns': info.st_mtime_ns} if hash_executable: result['sha256'] = b.digest(executable) if (birth != int((proc / 'stat').read_text().rsplit(') ', 1)[1].split()[19]) or command != (proc / 'cmdline').read_bytes() or uid != proc.stat().st_uid or scope == 'proc-exe' and executable != (proc / 'exe').resolve(strict=True)): raise RuntimeError('Desktop process changed during identity capture') return result class GPUOwnership: """Monitor owned GPU work plus explicitly approved existing desktop graphics. Unknown/new graphics, foreign compute (even an approved desktop PID), changed process identity or inaccessible NVML fail closed. Desktop clients are NEVER cleanup-owned. This is shared display monitoring, not GPU partitioning. """ class Process(ctypes.Structure): _fields_ = [('pid', ctypes.c_uint), ('usedGpuMemory', ctypes.c_ulonglong), ('gpuInstanceId', ctypes.c_uint), ('computeInstanceId', ctypes.c_uint)] def __init__(self, identity): if (not isinstance(identity, dict) or set(identity) != {'library', 'sha256', 'uuid', 'index', 'desktop_clients'} or identity['index'] != 0 or type(identity['index']) is not int): raise ValueError('Exact GPU0 NVML identity required') path = Path(identity['library']) if not path.is_absolute() or path != path.resolve() or b.digest(path) != identity['sha256']: raise ValueError('NVML library identity mismatch') self.identity = identity self.lib = ctypes.CDLL(str(path)) self.call('nvmlInit_v2') self.handle = ctypes.c_void_p() self.call('nvmlDeviceGetHandleByIndex_v2', ctypes.c_uint(0), ctypes.byref(self.handle)) uuid = ctypes.create_string_buffer(96) self.call('nvmlDeviceGetUUID', self.handle, uuid, ctypes.c_uint(len(uuid))) if uuid.value.decode() != identity['uuid']: raise RuntimeError('GPU UUID mismatch') clients = identity['desktop_clients'] if (not isinstance(clients, list) or len(clients) > 64 or any(not isinstance(x, dict) or type(x.get('pid')) is not int for x in clients) or [x['pid'] for x in clients] != sorted({x['pid'] for x in clients})): raise ValueError('Exact finite desktop client identities required') self.desktop = {x['pid']: x for x in clients} actual = self.clients() if actual['compute'] or actual['graphics'] != set(self.desktop): raise RuntimeError('GPU admission compute/approved-desktop mismatch') for pid, expected in self.desktop.items(): if desktop_identity(pid) != expected: raise RuntimeError('GPU desktop admission identity mismatch') def call(self, name, *args): code = getattr(self.lib, name)(*args) if code != 0: raise RuntimeError('NVML query failed: ' + name + ':' + str(code)) def clients(self): result = {} for kind, name in (('compute', 'nvmlDeviceGetComputeRunningProcesses_v2'), ('graphics', 'nvmlDeviceGetGraphicsRunningProcesses_v2')): count = ctypes.c_uint(1024) processes = (self.Process * 1024)() self.call(name, self.handle, ctypes.byref(count), processes) if count.value > 1024: raise RuntimeError('GPU ownership list overflow') result[kind] = {processes[i].pid for i in range(count.value)} return result def pids(self): clients = self.clients() return clients['compute'] | clients['graphics'] def check(self, owned, *, final=False): if b.digest(self.identity['library']) != self.identity['sha256']: raise RuntimeError('NVML drift') clients = self.clients() active = set(owned.discover()) graphics = clients['graphics'] desktop = graphics & set(self.desktop) for pid in desktop: expected = self.desktop[pid] if not final: expected = {k: v for k, v in expected.items() if k != 'sha256'} if desktop_identity(pid, hash_executable=final) != expected: raise RuntimeError('GPU desktop process identity changed') if (clients['compute'] - active or graphics - active - desktop or clients['compute'] & set(self.desktop) or final and (clients['compute'] or graphics - desktop)): raise RuntimeError('GPU ownership lost or residual task/foreign GPU client') return {'uuid': self.identity['uuid'], 'compute_pids': sorted(clients['compute']), 'approved_desktop_graphics_pids': sorted(desktop), 'task_graphics_pids': sorted(graphics - desktop), 'clean': not clients['compute'] and not (graphics - desktop), 'scope': 'owned-compute-and-graphics.plus-pinned-desktop-graphics-only'} def worker_payload(fd): """RootXY entrypoint-specific authenticated single-use inherited FD.""" with socket.socket(fileno=fd) as channel: channel.settimeout(15) pid, uid, _ = struct.unpack('3i', channel.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) command = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0') entry = command[1:] # only harmless interpreter flags may precede the actual entry script while entry and entry[0] in (b'-u', b'-B', b'-s', b'-E', b'-I'): entry = entry[1:] script_matches = bool(entry and Path(entry[0].decode()).is_absolute() and Path(entry[0].decode()) == ROOT / SCRIPT) if (not script_matches or Path(f'/proc/{pid}/exe').resolve() != Path('/usr/bin/python3.12') or not owner.ancestor(pid, os.getpid()) or uid != os.getuid() or not (b'--run' in entry or b'--prepare' in entry)): raise PermissionError('Authenticated RootXY supervisor required') channel.sendall(str(os.getpid()).encode() + b'\n') with channel.makefile('rb') as stream: data = stream.read(1 << 20) if len(data) >= 1 << 20: raise ValueError('Oversized RootXY handoff') payload = json.loads(data) if payload['launch_identity'] != launch_identity(): raise ValueError('RootXY launch drift') return payload def validate_completion(root, result, manifest, start): data = read(root / 'budget.json') expected = {'reserved_transitions': 1048576, 'reservations': 256, 'pending': None, 'updates': {'92511': 256}, 'validation_reserved_transitions': 806400, 'validation_attempted_control_calls': 5400, 'validation_completed_control_calls': 5400, 'validation_evaluations_started': ['analytic_zero', 'frozen_u256', 'u0064', 'u0128', 'u0192', 'u0256']} if any(data.get(k) != v or result.get('budget', {}).get(k) != v for k, v in expected.items()): raise ValueError('Incomplete optimization/validation debit evidence') if (data['clock']['start_s'] != start or data['clock']['deadline_s'] != start + 7200 or data['clock']['boot_id'] != b.boot_id()): raise ValueError('Worker changed supervisor lifetime') physics = result['physics'] attempted, completed, init, rollout, startup = [physics[k] for k in ( 'attempted_shared_calls', 'completed_shared_calls', 'initialization_attempted_shared_calls', 'rollout_attempted_shared_calls', 'startup_completed_without_task_audit')] if (any(type(x) is not int for x in (attempted, completed, init, rollout, startup)) or not 0 <= startup <= init <= 1000 or rollout != 67960 or attempted != completed or completed != init + rollout or completed > 73460): raise ValueError('Actual shared physics completion/cap evidence mismatch') safety = physics['safety'] gates = read(ROOT / manifest['training']['safety_config'])['gates'] values = [safety[k] for k in ('max_effort_velocity_ratio', 'max_joint_excess_rad', 'max_guide_vertical_N')] if (safety['physics_samples'] != completed - startup or safety['finite'] is not True or any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values) or values[0] > gates['normalized_effort_velocity_max'] or values[1] > gates['joint_limit_excess_rad_max'] or values[2] != 0): raise ValueError('Missing/failing global physical safety evidence') for label in ('analytic_zero', 'frozen_u256'): rows = read(root / ('validation_' + label + '.json'))['rows'] if (len(rows) != 128 or [r.get('env') for r in rows] != list(range(128)) or any(r.get('success') is not True or r.get('functional_audit', {}).get('sim_functional_passed') is not True for r in rows)): raise ValueError('Missing original baseline functional128/128 evidence') def commit_candidate(root, result, manifest, budget): """Independent validation and sole-authority last atomic commit; no worker lock.""" # Import CPU protocol only after worker exit/cleanup; never initialize CUDA. selection_path = ROOT / owner.PACKAGE / 'legs/contact_cycle_sim/rootxy_selection_v2.py' spec = importlib.util.spec_from_file_location('_rootxy_selection_cpu', selection_path) selection = importlib.util.module_from_spec(spec) spec.loader.exec_module(selection) if (result.get('status') != 'completed_provisional' or result.get('training_complete') is not True or result.get('accepted_skill') is not False or result.get('checkpoints_selectable') is not False or result.get('postclose_identity_verified') is not True): raise ValueError('No successful provisional worker result') ranked = [] for update in (64, 128, 192, 256): document = read(root / f'validation_u{update:04d}.json') rows = document['rows'] # Recheck original functional audit, not just worker's candidate label. if any(r.get('functional_audit', {}).get('sim_functional_passed') is not True for r in rows): raise ValueError('Missing original functional128/128 evidence') key = selection.candidate_key(rows, update) if key is not None: ranked.append((key, update)) candidate = result.get('candidate') if not ranked: if candidate is not None or result.get('outcome') != 'no_candidate': raise ValueError('Ineligible worker candidate') return None key, update = min(ranked) if (not isinstance(candidate, dict) or set(candidate) != {'path', 'sha256', 'update', 'selection_key'} or candidate['update'] != update or candidate['path'] != f'checkpoint_u{update:04d}.pt' or candidate['selection_key'] != list(key) or b.digest(root / candidate['path']) != candidate['sha256']): raise ValueError('Candidate selection/hash mismatch') lock = {'format': 'skillfactory.rootxy-supervisor-candidate.v2', 'candidate': candidate, 'manifest_file_sha256': b.digest(root / 'manifest.json'), 'accepted_skill': False, 'authority': 'waited_exit_postexit_identity_clean_owned_GPU_cleanup_original_eligibility', 'downstream_requires_separate_grant': True} budget.write(root / 'locked_candidate.json', lock, cleanup=True) return lock def supervise(args): start = b.boottime() # includes admission/startup; never reset in worker manifest, grant, policy = admission(args, start) mode = 'prepare' if args.prepare else 'run' if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'): raise RuntimeError('pidfd supervisor required') owned = owner.OwnedProcesses() gpu = GPUOwnership(grant['gpu_identity']) environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', CUDA_VISIBLE_DEVICES=grant['gpu_identity']['uuid']) owner.launch_command([], environment) if mode == 'prepare': b.durable_directory(args.output) lock = (args.output / (mode + '-claim.lock')).open('xb') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) budget = GuardBudget(args.output, start, mode) monitor = child = capture = channels = None errors, handlers = [], {} clean = waited = False result = None filesystem = None postexit = None def cleanup_pump(): if capture is not None: capture.pump() if monitor is not None: try: monitor.check() except BaseException as error: if not any(e.startswith('native_cleanup:') for e in errors): errors.append('native_cleanup:' + str(error)[-2048:]) try: monitor = owner.NativeMonitor(policy, SimpleNamespace(boottime=b.boottime, STORAGE_SCOPE=STORAGE_SCOPE)) budget.monitor = monitor handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True)) for s in (signal.SIGTERM, signal.SIGINT)} budget.write(args.output / (mode + '-reservation.json'), { 'profile': PROFILE, 'mode': mode, 'start_boottime': start, 'deadline_boottime': budget.deadline, 'boot_id': budget.boot, 'grant': grant, 'reserved_optimization_environment_transitions': 0 if mode == 'prepare' else 1048576, 'reserved_validation_environment_transitions': 0 if mode == 'prepare' else 806400, 'reserved_rollout_shared_calls': 0 if mode == 'prepare' else 72460, 'initialization_shared_calls_cap': 0 if mode == 'prepare' else 1000, 'no_refund': True, 'accepted_skill': False}) budget.write(args.output / (mode + '-manifest.json'), manifest) if mode == 'prepare': for relative, digest in manifest['sources'].items(): destination = args.output / 'source_snapshot' / relative destination.parent.mkdir(parents=True, exist_ok=True) data = (ROOT / relative).read_bytes() budget.check(len(data)) b.durable_write(destination, data) original_manifest = Path(args.manifest).read_bytes() budget.check(len(original_manifest)) b.durable_write(args.output / 'manifest.json', original_manifest) filesystem = owner.bound_filesystem_inputs(original_manifest) else: filesystem = owner.bound_filesystem_inputs((args.output / 'training-binding.json').read_bytes()) owner.verify_filesystem_inputs(filesystem) budget.write(args.output / (mode + '-grant.json'), grant) if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: raise RuntimeError('Subreaper setup failed') native = args.output / 'native' if native != native.resolve(): raise ValueError('Native redirect symlink forbidden') native.mkdir(exist_ok=True) for directory in redirects(args.output, mode).values(): Path(directory).mkdir(parents=True, exist_ok=False) environment.update(redirects(args.output, mode)) channels = socket.socketpair() parent, child_channel = channels child_channel.set_inheritable(True) budget.check() child = subprocess.Popen(owner.launch_command( [str(ROOT / SCRIPT), '--_worker-fd', str(child_channel.fileno())], environment), cwd=ROOT, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True, pass_fds=(child_channel.fileno(),)) try: owned.attach(child) except BaseException: owned.unknown = True raise child_channel.close() capture = owner.StdoutCapture(child.stdout, args.output / (mode + '-stdout.log'), b) budget.write(args.output / 'monitor-heartbeat.json', { 'boot_id': budget.boot, 'boottime': b.boottime(), 'external_growth': budget.external_growth}) owner.exchange_worker(child, parent, { 'mode': mode, 'output': str(args.output), 'start': start, 'boot_id': budget.boot, 'manifest_sha256': args.trusted_manifest_sha256, 'grant': grant, 'launch_identity': launch_identity()}, budget, capture, owned, monitor, b) next_heartbeat = 0 while child.poll() is None or not capture.eof: capture.check() budget.check() gpu.check(owned) if b.boottime() >= next_heartbeat: budget.write(args.output / 'monitor-heartbeat.json', { 'boot_id': budget.boot, 'boottime': b.boottime(), 'external_growth': budget.external_growth}) next_heartbeat = b.boottime() + .5 time.sleep(.01) child.wait(timeout=max(.01, budget.deadline - b.boottime())) waited = True result = read(args.output / (mode + '-worker-result.json')) if child.returncode != 0: raise RuntimeError('Worker nonzero exit') except BaseException as error: errors.append(str(error)[-8192:]) finally: try: if channels: for channel in channels: channel.close() if child is not None: if capture is None and child.stdout is not None: child.stdout.close() try: clean = owner.cleanup_group(child, budget.deadline, b, owned, cleanup_pump) child.wait(timeout=max(.01, budget.deadline - b.boottime())) waited = True except BaseException as error: errors.append('cleanup:' + str(error)[-2048:]) if capture is not None: try: capture.close() if capture.discarded or capture.error or not capture.eof: raise RuntimeError('Incomplete/overflowed stdout') except BaseException as error: errors.append(str(error)[-2048:]) try: if not clean or not waited or child is None or child.returncode != 0 or owned.unknown: raise RuntimeError('No clean independently waited worker exit') gpu.check(owned, final=True) verify_sources(manifest) if filesystem is None: raise RuntimeError('Missing prelaunch filesystem identity') postexit = owner.verify_filesystem_inputs(filesystem) binding_files = owner.bound_filesystem_inputs((args.output / 'training-binding.json').read_bytes()) owner.verify_filesystem_inputs(binding_files) if result is None or result.get('postclose_identity_verified') is not True: raise RuntimeError('Missing postclose same-object verification') if mode == 'prepare': if result.get('status') != 'prepared_config_only' or result.get('physics_calls') != 0: raise RuntimeError('PREP incomplete/config-only violation') else: if result.get('status') != 'completed_provisional' or result.get('training_complete') is not True: raise RuntimeError('Training incomplete') validate_completion(args.output, result, manifest, start) budget.check(cleanup=True) monitor.check(force=True) except BaseException as error: errors.append(str(error)[-4096:]) summary = {'success': not errors, 'mode': mode, 'cleanup_clean': clean, 'waited_exit': waited, 'returncode': None if child is None else child.returncode, 'errors': errors, 'accepted_skill': False, 'postexit': postexit, 'owned': sorted(owned.known), 'storage': None if monitor is None else monitor.report()} budget.write(args.output / (mode + '-supervisor.json'), summary, cleanup=True) if not errors and mode == 'prepare': prepared = read(args.output / 'config-preparation-provisional.json') prepared.update(supervisor_sha256=b.digest(args.output / 'prepare-supervisor.json'), worker_result_sha256=b.digest(args.output / 'prepare-worker-result.json'), postexit_verified=True) try: budget.write(args.output / 'preparation.json', prepared, cleanup=True) except BaseException: (args.output / 'preparation.json').unlink(missing_ok=True) raise if not errors and mode == 'run': # Last operation capable of committing success. Failure leaves no candidate. try: commit_candidate(args.output, result, manifest, budget) except BaseException as error: (args.output / 'locked_candidate.json').unlink(missing_ok=True) summary.update(success=False, errors=[*errors, 'candidate_commit:' + str(error)[-2048:]]) budget.write(args.output / (mode + '-supervisor.json'), summary, cleanup=True) raise finally: for sig, handler in handlers.items(): signal.signal(sig, handler) lock.close() return 0 if not errors else 1 def preimport_binding_runtime(binding): """Finite binding-required imports, before config/inventory; no optional skips.""" for name in dict.fromkeys((*binding.REQUIRED_RUNTIME, *binding.ISAAC_PINS)): importlib.import_module(name) class RuntimeAPI: """Actual Kit recipe; callable only from authenticated worker handoff.""" def __init__(self, payload, budget, guard): self.payload, self.budget, self.guard = payload, budget, guard self.output = Path(payload['output']) self.mode = payload['mode'] self.launcher = self.env = self.cfg = self.ledger = None self.started = False self.physics_hooks = None self.manifest = read(self.output / 'manifest.json', payload['manifest_sha256']) self.expected = None def start(self): from isaaclab.app import AppLauncher app_source = self.payload['launch_identity']['app_launcher'] if (AppLauncher.__module__ != 'isaaclab.app.app_launcher' or Path(sys.modules[AppLauncher.__module__].__file__).resolve() != Path(app_source['path']) or b.digest(app_source['path']) != app_source['sha256']): raise ValueError('Actual installed AppLauncher identity mismatch') # Retain launcher even if initialization fails after creating its app. self.launcher = AppLauncher.__new__(AppLauncher) AppLauncher.__init__(self.launcher, headless=True, device='cuda:0', fast_shutdown=True, kit_args=f'--/log/file={self.output / "native" / self.mode / "kit.log"}') self.started = True for name in ('rootxy_training_v2', 'rootxy_runner_v2', 'rootxy_binding_v2', 'rootxy_runtime_v2'): setattr(self, name, importlib.import_module(owner.LEG + name)) self.p = self.rootxy_training_v2 self.binding = self.rootxy_binding_v2 eb = importlib.import_module(owner.LEG + 'evaluation_binding') self.eb = eb for name in self.binding.ROLES: if name != '__main__': importlib.import_module(name) # Do not depend on an evaluation recorder to trigger lazy backends. preimport_binding_runtime(eb) # Match the finite installed constructor preimports, never a whole interpreter scan. owner.preimport_replicator() owner.preimport_constructor_modules(eb) self.constructor_export, self.constructor_metadata = owner.physx_constructor_export(eb, resolve=True) self.cfg_class = importlib.import_module(owner.TASK + 'rootxy_cfg_v2').RootXYContactCycleEnvCfg self.environment_class = importlib.import_module(owner.TASK + 'env').SimContactCycleEnv def configure(self): self.cfg = self.cfg_class() self.cfg.scene.num_envs = 128 self.cfg.seed = 92511 self.cfg.sim.device = 'cuda:0' # This helper only verifies installed stepping and explicitly forwards # fixed constructor video config fields; no evaluation environment created. owner.preforward_video_recorder(self.cfg, importlib.import_module(owner.TASK + 'evaluation_env')) owner.preimport_config_references(self.cfg, self.eb) self.binding.normalize_config(self.cfg) # exact128-env PREconstruction only import torch torch.cuda.init() torch.cuda.get_rng_state(torch.device('cuda:0')) if self.mode == 'prepare': self.inventory = self.binding.propose_inventory(ROOT, self.cfg, owner.proposed_native_layouts) self.expected = self.binding.snapshot(ROOT, self.output, self.cfg, self.inventory, self.manifest) else: self.expected = read(self.output / 'training-binding.json', self.payload['grant']['binding_sha256']) self.inventory = self.expected['dependency_inventory'] if self.mode == 'run': outer = self class Ledger(self.p.RootXYBudgetLedger): def guard(self, reserve_bytes=0): outer.guard() outer.budget.check(reserve_bytes) return super().guard(reserve_bytes=reserve_bytes) self.ledger = Ledger(self.output, self.manifest, launch_start_s=self.payload['start']) self.ledger.__enter__() @contextmanager def physics(self): from isaaclab.sim import SimulationContext with self.rootxy_runtime_v2.RootXYPhysicsHooks( SimulationContext, self.environment_class, self.guard, lambda value: self.budget.write(self.output / 'physics-counts.json', value), preparation=self.mode == 'prepare') as hooks: # Config construction/normalization itself is under zero-physics PREP hooks. self.configure() yield hooks def verify(self): self.budget.check(cleanup=True) verify_sources(self.manifest) if self.cfg is None or self.expected is None: raise RuntimeError('Startup did not establish actual config identity') target, metadata = owner.physx_constructor_export(self.eb, expected=self.constructor_export) if metadata != self.constructor_metadata: raise ValueError('Constructor export drift') self.binding.verify(self.expected, root=ROOT, output=self.output, cfg=self.cfg, inventory=self.inventory, manifest=self.manifest) if self.env is not None and getattr(self.env, 'cfg', None) is not self.cfg: raise ValueError('Constructor substituted actual config object') def prepare(self): self.budget.write(self.output / 'training-binding.json', self.expected) result = {'format': PREPARATION_FORMAT, 'status': 'prepared_config_only', 'manifest_sha256': self.payload['manifest_sha256'], 'binding_sha256': b.digest(self.output / 'training-binding.json'), 'artifact_root': str(self.output), 'physics_calls': 0, 'num_envs': 128, 'runtime_qualified': False, 'p2_decision': 'pending', 'accepted_skill': False, 'recipe': {'device': 'cuda:0', 'train_seed': 92511, 'development_seed': 93511, 'environment_constructed': False, 'inference_executed': False, 'profile': PROFILE, 'hooks': self.binding.HOOKS}, 'ready_gaps': ['parent review of actual config-only binding', 'separate approved P2 decision and fresh run grant', 'immediate postconstructor actual-used identity and reset safety qualification']} self.budget.write(self.output / 'config-preparation-provisional.json', result) return result def construct(self): self.guard() self.env = self.environment_class.__new__(self.environment_class) self.environment_class.__init__(self.env, cfg=self.cfg) def check_actual(self): importlib.import_module(owner.TASK + 'rootxy_cfg_v2').verify_actual_config(self.env) if self.env._physics_handles_decimation: raise RuntimeError('Require scalar 500Hz physics stepping') def safety(self): from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit gates = read(ROOT / self.p.PILOT['safety_config'])['gates'] return PhysicalSafetyAudit(self.env, gates) def learn(self): import torch from audit_contact_cycle_functional import evaluate trace_path = ROOT / self.p.replay.CANDIDATE trace_path = trace_path.parent / 'validation_u0256_trace.pt' if b.digest(trace_path) != self.manifest['parity_trace']['sha256']: raise ValueError('Saved CPU parity trace drift') trace = torch.load(trace_path, map_location='cpu', weights_only=True) observations = next(iter(trace.values()))['actor_input'][::100, 0].contiguous() del trace def guard(): self.guard() self.physics_hooks.safety_boundary() def before_control(kind): if kind not in ('optimization', 'validation'): raise ValueError('Unknown control budget kind') guard() self.physics_hooks.mode = 'rollout' def after_control(kind): self.physics_hooks.mode = 'initialization' guard() hooks = self.rootxy_runner_v2.RuntimeHooks( admit=lambda manifest: {'manifest_sha256': self.p.digest(manifest), 'p2_review_approved': True, 'sole_cleanup_owner': True, 'native_monitor_active': True}, guard=guard, verify_identity=self.verify, before_control=before_control, after_control=after_control, progress=lambda value: guard(), finish=lambda value: guard()) return self.rootxy_runner_v2.execute(self.env, ROOT, self.output, self.manifest, self.ledger, observations, hooks, evaluate) def close_env(self): if self.env is not None: self.env.close() def close_app(self, code): error = None try: if self.ledger is not None: self.ledger.__exit__() except BaseException as caught: error = caught app = None if self.launcher is None else getattr(self.launcher, '_app', None) if app is not None: # Fast shutdown may exit instead of returning: never pass zero after # another teardown failure and rely on later Python unwinding. app.close(exit_code=1 if error is not None else code) if error is not None: raise error def run_worker(fd): payload = worker_payload(fd) # no unauthenticated dictionary entrypoint if payload['mode'] not in ('prepare', 'run') or payload['boot_id'] != b.boot_id(): raise PermissionError('Invalid authenticated RootXY handoff') root = checked_root(payload['output']) reservation = read(root / (payload['mode'] + '-reservation.json')) if (reservation['grant'] != payload['grant'] or reservation['start_boottime'] != payload['start'] or reservation['boot_id'] != payload['boot_id']): raise PermissionError('RootXY reservation mismatch') budget = GuardBudget(root, payload['start'], payload['mode']) def guard(): heartbeat = read(root / 'monitor-heartbeat.json') if (heartbeat['boot_id'] != b.boot_id() or not 0 <= b.boottime() - heartbeat['boottime'] <= 5): raise RuntimeError('Native monitor heartbeat missing/stale') budget.external_growth = heartbeat['external_growth'] budget.check() handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True)) for s in (signal.SIGTERM, signal.SIGINT)} # Pure hook module is loaded under a private identity before Kit, not a fake # SkillFactory package. Actual runtime imports after AppLauncher use real paths. spec = importlib.util.spec_from_file_location('_rootxy_lifecycle_cpu', ROOT / owner.PACKAGE / 'legs/contact_cycle_sim/rootxy_runtime_v2.py') runtime = importlib.util.module_from_spec(spec) spec.loader.exec_module(runtime) api = RuntimeAPI(payload, budget, guard) verify_sources(api.manifest) try: runtime.worker_lifecycle(api, prepare=payload['mode'] == 'prepare', guard=guard, verify=api.verify, publish=lambda value: budget.write( root / (payload['mode'] + '-worker-result.json'), value, cleanup=True)) return 0 finally: for sig, handler in handlers.items(): signal.signal(sig, handler)