9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
872 lines
48 KiB
Python
872 lines
48 KiB
Python
"""Dedicated RootXY supervisor adapter. Importing this module never starts Kit.
|
|
|
|
Ownership/native monitoring/launch transport reuse reviewed evaluator helpers;
|
|
no evaluation grant, protocol, ledger, checkpoint or candidate authority is reused.
|
|
"""
|
|
import argparse
|
|
import ctypes
|
|
from contextlib import contextmanager
|
|
import fcntl
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import math
|
|
import os
|
|
import signal
|
|
import socket
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import evaluate_cycle_sim_residual as owner
|
|
|
|
ROOT = owner.ROOT
|
|
SCRIPT = 'scripts/train_cycle_sim_rootxy_v2.py'
|
|
PROFILE = 'rootxy-v2-7200s-1GiB-73460calls'
|
|
TASK_ID = 'SkillFactory-Go2FL-ContactCycleSimRootXY-v2'
|
|
PREREG_FORMAT = 'skillfactory.contact-cycle-sim-rootxy-preregistration.v2'
|
|
GRANT_FORMAT = 'skillfactory.rootxy-parent-grant.v2'
|
|
POLICY_FORMAT = 'skillfactory.rootxy-native-policy.v2'
|
|
PREPARATION_FORMAT = 'skillfactory.rootxy-config-preparation.v2'
|
|
STORAGE_SCOPE = 'rootxy-new-owned.native-monitored-stop.v2'
|
|
DECISION_FORMAT = 'skillfactory.rootxy-p2-decision.v2'
|
|
CAP = 1 << 30
|
|
HEADROOM = 16 << 20
|
|
STDOUT = 1 << 20
|
|
NATIVE_LIFETIME = 10800
|
|
b = owner.cpu_module('evaluation_budget')
|
|
|
|
|
|
def canonical(value):
|
|
return b.encode(value)
|
|
|
|
|
|
def sha(value):
|
|
return hashlib.sha256(canonical(value)).hexdigest()
|
|
|
|
|
|
def read(path, expected=None):
|
|
path = Path(path)
|
|
if not path.is_file() or path.is_symlink() or path.stat().st_size > 32 << 20:
|
|
raise ValueError('Missing/unsafe/oversized RootXY input: ' + str(path))
|
|
data = path.read_bytes()
|
|
if expected is not None:
|
|
owner.read_trusted(path, expected)
|
|
return json.loads(data)
|
|
|
|
|
|
def checked_root(path):
|
|
root = Path(path)
|
|
if (not root.is_absolute() or root != root.resolve()
|
|
or root.is_relative_to(ROOT) and not root.is_relative_to(ROOT / 'logs/contact_cycle/rootxy_v2')
|
|
or root.is_relative_to(ROOT / 'releases')
|
|
or root.is_relative_to(ROOT / 'logs/contact_cycle/sim_residual_training_v1')):
|
|
raise ValueError('Canonical new isolated RootXY root required')
|
|
return root
|
|
|
|
|
|
def verify_sources(manifest):
|
|
if (manifest.get('format') != PREREG_FORMAT or manifest.get('task_id') != TASK_ID
|
|
or manifest.get('launch_integration_complete') is not True
|
|
or manifest.get('accepted_skill') is not False):
|
|
raise ValueError('New RootXY preregistration required')
|
|
for relative, expected in manifest['sources'].items():
|
|
path = ROOT / relative
|
|
if (Path(relative).is_absolute() or '..' in Path(relative).parts or path != path.resolve()
|
|
or b.digest(path) != expected):
|
|
raise ValueError('RootXY source drift: ' + relative)
|
|
cfg = manifest['training']
|
|
fixed = {'seeds': [92511], 'validation_seed': 93511, 'num_envs': 128,
|
|
'max_total_wall_s': 7200, 'cleanup_reserve_s': 120, 'max_artifact_bytes': CAP,
|
|
'max_validation_transitions': 806400, 'validation_updates': [64, 128, 192, 256]}
|
|
if any(cfg.get(k) != v for k, v in fixed.items()):
|
|
raise ValueError('RootXY resource contract changed')
|
|
|
|
|
|
def launch_identity():
|
|
app_path = Path('/home/cen/IsaacLab/source/isaaclab/isaaclab/app/app_launcher.py')
|
|
return {'app_launcher': {'path': str(app_path), 'sha256': b.digest(app_path)}, 'format': 'skillfactory.rootxy-launch-identity.v2',
|
|
'transport': owner.launch_identity(), 'profile': PROFILE, 'task_id': TASK_ID,
|
|
'entrypoint': {'path': str(ROOT / SCRIPT), 'sha256': b.digest(ROOT / SCRIPT)},
|
|
'adapter': {'path': str(Path(__file__).resolve()), 'sha256': b.digest(__file__)}}
|
|
|
|
|
|
def redirects(root, mode):
|
|
return owner.native_redirects(root / 'native' / mode, preparation=True)
|
|
|
|
|
|
def policy_check(args, now):
|
|
policy = read(args.storage_policy, args.trusted_storage_policy_sha256)
|
|
root = args.output
|
|
fields = {'format', 'profile', 'artifact_root', 'limit_bytes', 'native_paths', 'shared_native_paths',
|
|
'task_owned_cache_paths', 'free_space_paths', 'free_space_mount_identities', 'mount_identity',
|
|
'check_interval_seconds', 'max_native_growth_bytes', 'min_free_bytes', 'coverage_gaps',
|
|
'native_hard_cap', 'boot_id', 'issued_boottime', 'not_after_boottime', 'lifetime_seconds'}
|
|
duration = policy.get('lifetime_seconds')
|
|
if (set(policy) != fields or policy['format'] != POLICY_FORMAT or policy['profile'] != PROFILE
|
|
or policy['artifact_root'] != str(root) or policy['limit_bytes'] != CAP
|
|
or policy['native_paths'] != owner.native_paths(root)
|
|
or policy['shared_native_paths'] != owner.shared_native_paths()
|
|
or policy['task_owned_cache_paths'] != sorted([*redirects(root, 'prepare').values(),
|
|
*redirects(root, 'run').values()])
|
|
or policy['free_space_paths'] != owner.free_space_paths(root)
|
|
or policy['free_space_mount_identities'] != {
|
|
p: owner.mount_identity(Path(p)) for p in owner.free_space_paths(root)}
|
|
or policy['mount_identity'] != owner.mount_identity(root)
|
|
or policy['coverage_gaps'] != list(owner.NATIVE_GAPS) or policy['native_hard_cap'] is not False
|
|
or policy['boot_id'] != b.boot_id() or type(duration) is not int or not 7200 <= duration <= 10800
|
|
or type(policy['issued_boottime']) not in (int, float)
|
|
or not 0 <= policy['issued_boottime'] <= now
|
|
or policy['not_after_boottime'] != policy['issued_boottime'] + duration
|
|
or type(policy['check_interval_seconds']) not in (int, float)
|
|
or not 0 < policy['check_interval_seconds'] <= 1
|
|
or type(policy['max_native_growth_bytes']) is not int
|
|
or not 0 < policy['max_native_growth_bytes'] <= CAP
|
|
or type(policy['min_free_bytes']) is not int or policy['min_free_bytes'] < 2 * CAP):
|
|
raise PermissionError('Exact RootXY native MONITOR policy required (not quota)')
|
|
return policy
|
|
|
|
|
|
def admission(args, now):
|
|
args.output = checked_root(args.output)
|
|
mode = 'prepare' if args.prepare else 'run'
|
|
required = ('manifest', 'trusted_manifest_sha256', 'authorization', 'trusted_authorization_sha256',
|
|
'storage_policy', 'trusted_storage_policy_sha256')
|
|
if any(getattr(args, k, None) is None for k in required):
|
|
raise PermissionError('Fresh exact RootXY manifest/native-policy/parent grant required')
|
|
manifest = read(args.manifest, args.trusted_manifest_sha256)
|
|
verify_sources(manifest)
|
|
grant = read(args.authorization, args.trusted_authorization_sha256)
|
|
fields = {'format', 'profile', 'task_id', 'mode', 'artifact_root', 'manifest_sha256', 'storage_policy_sha256',
|
|
'launch_identity_sha256', 'boot_id', 'not_after_boottime', 'parent_review',
|
|
'preparation_sha256', 'binding_sha256', 'p2_decision_sha256', 'gpu_identity'}
|
|
if (set(grant) != fields or grant['format'] != GRANT_FORMAT or grant['profile'] != PROFILE
|
|
or grant['task_id'] != TASK_ID or grant['mode'] != mode or grant['artifact_root'] != str(args.output)
|
|
or grant['manifest_sha256'] != args.trusted_manifest_sha256
|
|
or grant['storage_policy_sha256'] != args.trusted_storage_policy_sha256
|
|
or grant['launch_identity_sha256'] != sha(launch_identity())
|
|
or grant['boot_id'] != b.boot_id() or type(grant['not_after_boottime']) not in (int, float)
|
|
or not now < grant['not_after_boottime'] <= now + 3600
|
|
or type(grant['parent_review']) is not str or not grant['parent_review'].strip()):
|
|
raise PermissionError('RootXY parent grant identity/expiry mismatch')
|
|
if mode == 'prepare':
|
|
if any(grant[k] is not None for k in ('preparation_sha256', 'binding_sha256', 'p2_decision_sha256')):
|
|
raise PermissionError('PREP has pending P2/config identity only, not runtime approval')
|
|
if args.output.exists():
|
|
raise FileExistsError('PREP requires a NEW root; no retry')
|
|
else:
|
|
for field in ('preparation_sha256', 'binding_sha256', 'p2_decision_sha256'):
|
|
value = grant[field]
|
|
if (type(value) is not str or len(value) != 64
|
|
or any(c not in '0123456789abcdef' for c in value)):
|
|
raise PermissionError('RUN requires an exact independently approved ' + field)
|
|
if any((args.output / name).exists() for name in (
|
|
'run-claim.lock', 'launch_claim.json', 'budget.json', 'budget.lock', 'locked_candidate.json',
|
|
'run-worker-result.json', 'run-supervisor.json')):
|
|
raise FileExistsError('Fresh-only RootXY RUN; no prior trial or candidate artifacts')
|
|
preparation = read(args.output / 'preparation.json', grant['preparation_sha256'])
|
|
binding = read(args.output / 'training-binding.json', grant['binding_sha256'])
|
|
decision = read(args.p2_decision, grant['p2_decision_sha256'])
|
|
if (preparation.get('format') != PREPARATION_FORMAT or preparation.get('status') != 'prepared_config_only'
|
|
or preparation.get('manifest_sha256') != args.trusted_manifest_sha256
|
|
or preparation.get('binding_sha256') != grant['binding_sha256']
|
|
or preparation.get('artifact_root') != str(args.output)
|
|
or preparation.get('physics_calls') != 0 or preparation.get('runtime_qualified') is not False
|
|
or binding.get('format') != 'skillfactory.rootxy-training-binding.v2'
|
|
or binding.get('artifact_root') != str(args.output) or binding.get('profile') != PROFILE):
|
|
raise PermissionError('Exact config-only preparation/binding required')
|
|
prep_supervisor = read(args.output / 'prepare-supervisor.json', preparation['supervisor_sha256'])
|
|
read(args.output / 'prepare-worker-result.json', preparation['worker_result_sha256'])
|
|
if (preparation.get('postexit_verified') is not True
|
|
or prep_supervisor.get('success') is not True or prep_supervisor.get('cleanup_clean') is not True):
|
|
raise PermissionError('PREP needs independent clean waited-exit completion')
|
|
if (set(decision) != {'format', 'task_id', 'artifact_root', 'manifest_sha256', 'preparation_sha256',
|
|
'binding_sha256', 'approved', 'parent_review', 'p2_evidence'}
|
|
or decision['format'] != DECISION_FORMAT or decision['task_id'] != TASK_ID
|
|
or decision['artifact_root'] != str(args.output) or decision['approved'] is not True
|
|
or decision['manifest_sha256'] != args.trusted_manifest_sha256
|
|
or decision['preparation_sha256'] != grant['preparation_sha256']
|
|
or decision['binding_sha256'] != grant['binding_sha256']
|
|
or not isinstance(decision['parent_review'], str) or not decision['parent_review'].strip()
|
|
or not isinstance(decision['p2_evidence'], dict) or not decision['p2_evidence']):
|
|
raise PermissionError('Separate parent-approved P2 evidence/feasibility decision required')
|
|
for path, digest in decision['p2_evidence'].items():
|
|
owner.read_trusted(path, digest)
|
|
for relative, digest in manifest['sources'].items():
|
|
if b.digest(args.output / 'source_snapshot' / relative) != digest:
|
|
raise ValueError('RootXY preparation snapshot drift')
|
|
policy = policy_check(args, now)
|
|
total = 600 if mode == 'prepare' else 7200
|
|
if policy['not_after_boottime'] < max(grant['not_after_boottime'], now + total + 120):
|
|
raise PermissionError('Native policy must cover admission, full lifetime and headroom')
|
|
return manifest, grant, policy
|
|
|
|
|
|
class GuardBudget:
|
|
"""RootXY cap/deadline facade for reviewed ownership and stdout transport."""
|
|
def __init__(self, root, start, mode, monitor=None):
|
|
self.root, self.start, self.mode, self.monitor = Path(root), start, mode, monitor
|
|
self.boot = b.boot_id()
|
|
self.deadline = start + (600 if mode == 'prepare' else 7200)
|
|
self.reservation = {'deadline_boottime': self.deadline}
|
|
self.interrupted = False
|
|
self.external_growth = 0
|
|
|
|
def check(self, pending=0, *, cleanup=False):
|
|
if (self.interrupted and not cleanup or b.boot_id() != self.boot
|
|
or not self.start <= b.boottime() < self.deadline - (0 if cleanup else 120)):
|
|
raise RuntimeError('RootXY cancelled/deadline/boot mismatch')
|
|
if self.monitor is not None:
|
|
self.monitor.check()
|
|
self.external_growth = sum(max(0, size - self.monitor.baseline[path])
|
|
for path, size in self.monitor.latest.items()
|
|
if not Path(path).is_relative_to(self.root))
|
|
limit = CAP - (0 if cleanup else HEADROOM + STDOUT)
|
|
if b.artifact_size(self.root) + self.external_growth + pending > limit:
|
|
raise RuntimeError('RootXY aggregate new-owned/native storage cutoff')
|
|
|
|
def write(self, path, value, *, cleanup=False):
|
|
path = Path(path)
|
|
if path.parent != path.parent.resolve() or not path.is_relative_to(self.root) or path.is_symlink():
|
|
raise ValueError('Unsafe RootXY publication path')
|
|
data = canonical(value)
|
|
if len(data) > 16 << 20:
|
|
raise RuntimeError('Bounded RootXY publication exceeded')
|
|
self.check(len(data), cleanup=cleanup)
|
|
temporary = path.with_name(path.name + '.pending')
|
|
b.durable_write(temporary, data)
|
|
os.replace(temporary, path)
|
|
b.sync_directory(path.parent)
|
|
|
|
|
|
def desktop_identity(pid, *, hash_executable=True):
|
|
"""Pin a pre-existing graphics client, never infer permission from its name."""
|
|
if type(pid) is not int or pid <= 0:
|
|
raise ValueError('Invalid desktop PID')
|
|
proc = Path('/proc') / str(pid)
|
|
birth = int((proc / 'stat').read_text().rsplit(') ', 1)[1].split()[19])
|
|
uid = proc.stat().st_uid
|
|
command = (proc / 'cmdline').read_bytes()
|
|
scope = 'proc-exe'
|
|
try:
|
|
executable = (proc / 'exe').resolve(strict=True)
|
|
except PermissionError:
|
|
# Ubuntu's root-owned display server hides /proc/PID/exe from this user.
|
|
# Explicit, narrowly scoped public-metadata identity, NOT executable proof.
|
|
# Still requires a parent-pinned PID/birth/UID/argv and graphics-only NVML.
|
|
if (uid != 0 or command.split(b'\0')[0] != b'/usr/lib/xorg/Xorg'
|
|
or (proc / 'comm').read_text().strip() != 'Xorg'):
|
|
raise
|
|
executable = Path('/usr/lib/xorg/Xorg').resolve(strict=True)
|
|
scope = 'root-Xorg-public-metadata; actual-executable-unverified'
|
|
info = executable.stat()
|
|
result = {'pid': pid, 'starttime': birth, 'uid': uid,
|
|
'command_sha256': hashlib.sha256(command).hexdigest(),
|
|
'executable_identity_scope': scope, 'executable': str(executable),
|
|
'device': info.st_dev, 'inode': info.st_ino, 'size': info.st_size,
|
|
'mtime_ns': info.st_mtime_ns}
|
|
if hash_executable:
|
|
result['sha256'] = b.digest(executable)
|
|
if (birth != int((proc / 'stat').read_text().rsplit(') ', 1)[1].split()[19])
|
|
or command != (proc / 'cmdline').read_bytes() or uid != proc.stat().st_uid
|
|
or scope == 'proc-exe' and executable != (proc / 'exe').resolve(strict=True)):
|
|
raise RuntimeError('Desktop process changed during identity capture')
|
|
return result
|
|
|
|
|
|
class GPUOwnership:
|
|
"""Monitor owned GPU work plus explicitly approved existing desktop graphics.
|
|
|
|
Unknown/new graphics, foreign compute (even an approved desktop PID), changed
|
|
process identity or inaccessible NVML fail closed. Desktop clients are NEVER
|
|
cleanup-owned. This is shared display monitoring, not GPU partitioning.
|
|
"""
|
|
class Process(ctypes.Structure):
|
|
_fields_ = [('pid', ctypes.c_uint), ('usedGpuMemory', ctypes.c_ulonglong),
|
|
('gpuInstanceId', ctypes.c_uint), ('computeInstanceId', ctypes.c_uint)]
|
|
|
|
def __init__(self, identity):
|
|
if (not isinstance(identity, dict) or set(identity) != {'library', 'sha256', 'uuid', 'index', 'desktop_clients'}
|
|
or identity['index'] != 0 or type(identity['index']) is not int):
|
|
raise ValueError('Exact GPU0 NVML identity required')
|
|
path = Path(identity['library'])
|
|
if not path.is_absolute() or path != path.resolve() or b.digest(path) != identity['sha256']:
|
|
raise ValueError('NVML library identity mismatch')
|
|
self.identity = identity
|
|
self.lib = ctypes.CDLL(str(path))
|
|
self.call('nvmlInit_v2')
|
|
self.handle = ctypes.c_void_p()
|
|
self.call('nvmlDeviceGetHandleByIndex_v2', ctypes.c_uint(0), ctypes.byref(self.handle))
|
|
uuid = ctypes.create_string_buffer(96)
|
|
self.call('nvmlDeviceGetUUID', self.handle, uuid, ctypes.c_uint(len(uuid)))
|
|
if uuid.value.decode() != identity['uuid']:
|
|
raise RuntimeError('GPU UUID mismatch')
|
|
clients = identity['desktop_clients']
|
|
if (not isinstance(clients, list) or len(clients) > 64
|
|
or any(not isinstance(x, dict) or type(x.get('pid')) is not int for x in clients)
|
|
or [x['pid'] for x in clients] != sorted({x['pid'] for x in clients})):
|
|
raise ValueError('Exact finite desktop client identities required')
|
|
self.desktop = {x['pid']: x for x in clients}
|
|
actual = self.clients()
|
|
if actual['compute'] or actual['graphics'] != set(self.desktop):
|
|
raise RuntimeError('GPU admission compute/approved-desktop mismatch')
|
|
for pid, expected in self.desktop.items():
|
|
if desktop_identity(pid) != expected:
|
|
raise RuntimeError('GPU desktop admission identity mismatch')
|
|
|
|
def call(self, name, *args):
|
|
code = getattr(self.lib, name)(*args)
|
|
if code != 0:
|
|
raise RuntimeError('NVML query failed: ' + name + ':' + str(code))
|
|
|
|
def clients(self):
|
|
result = {}
|
|
for kind, name in (('compute', 'nvmlDeviceGetComputeRunningProcesses_v2'),
|
|
('graphics', 'nvmlDeviceGetGraphicsRunningProcesses_v2')):
|
|
count = ctypes.c_uint(1024)
|
|
processes = (self.Process * 1024)()
|
|
self.call(name, self.handle, ctypes.byref(count), processes)
|
|
if count.value > 1024:
|
|
raise RuntimeError('GPU ownership list overflow')
|
|
result[kind] = {processes[i].pid for i in range(count.value)}
|
|
return result
|
|
|
|
def pids(self):
|
|
clients = self.clients()
|
|
return clients['compute'] | clients['graphics']
|
|
|
|
def check(self, owned, *, final=False):
|
|
if b.digest(self.identity['library']) != self.identity['sha256']:
|
|
raise RuntimeError('NVML drift')
|
|
clients = self.clients()
|
|
active = set(owned.discover())
|
|
graphics = clients['graphics']
|
|
desktop = graphics & set(self.desktop)
|
|
for pid in desktop:
|
|
expected = self.desktop[pid]
|
|
if not final:
|
|
expected = {k: v for k, v in expected.items() if k != 'sha256'}
|
|
if desktop_identity(pid, hash_executable=final) != expected:
|
|
raise RuntimeError('GPU desktop process identity changed')
|
|
if (clients['compute'] - active or graphics - active - desktop
|
|
or clients['compute'] & set(self.desktop)
|
|
or final and (clients['compute'] or graphics - desktop)):
|
|
raise RuntimeError('GPU ownership lost or residual task/foreign GPU client')
|
|
return {'uuid': self.identity['uuid'], 'compute_pids': sorted(clients['compute']),
|
|
'approved_desktop_graphics_pids': sorted(desktop),
|
|
'task_graphics_pids': sorted(graphics - desktop),
|
|
'clean': not clients['compute'] and not (graphics - desktop),
|
|
'scope': 'owned-compute-and-graphics.plus-pinned-desktop-graphics-only'}
|
|
|
|
|
|
def worker_payload(fd):
|
|
"""RootXY entrypoint-specific authenticated single-use inherited FD."""
|
|
with socket.socket(fileno=fd) as channel:
|
|
channel.settimeout(15)
|
|
pid, uid, _ = struct.unpack('3i', channel.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
|
|
command = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0')
|
|
entry = command[1:] # only harmless interpreter flags may precede the actual entry script
|
|
while entry and entry[0] in (b'-u', b'-B', b'-s', b'-E', b'-I'):
|
|
entry = entry[1:]
|
|
script_matches = bool(entry and Path(entry[0].decode()).is_absolute()
|
|
and Path(entry[0].decode()) == ROOT / SCRIPT)
|
|
if (not script_matches or Path(f'/proc/{pid}/exe').resolve() != Path('/usr/bin/python3.12')
|
|
or not owner.ancestor(pid, os.getpid()) or uid != os.getuid()
|
|
or not (b'--run' in entry or b'--prepare' in entry)):
|
|
raise PermissionError('Authenticated RootXY supervisor required')
|
|
channel.sendall(str(os.getpid()).encode() + b'\n')
|
|
with channel.makefile('rb') as stream:
|
|
data = stream.read(1 << 20)
|
|
if len(data) >= 1 << 20:
|
|
raise ValueError('Oversized RootXY handoff')
|
|
payload = json.loads(data)
|
|
if payload['launch_identity'] != launch_identity():
|
|
raise ValueError('RootXY launch drift')
|
|
return payload
|
|
|
|
|
|
def validate_completion(root, result, manifest, start):
|
|
data = read(root / 'budget.json')
|
|
expected = {'reserved_transitions': 1048576, 'reservations': 256, 'pending': None,
|
|
'updates': {'92511': 256}, 'validation_reserved_transitions': 806400,
|
|
'validation_attempted_control_calls': 5400, 'validation_completed_control_calls': 5400,
|
|
'validation_evaluations_started': ['analytic_zero', 'frozen_u256', 'u0064', 'u0128',
|
|
'u0192', 'u0256']}
|
|
if any(data.get(k) != v or result.get('budget', {}).get(k) != v for k, v in expected.items()):
|
|
raise ValueError('Incomplete optimization/validation debit evidence')
|
|
if (data['clock']['start_s'] != start or data['clock']['deadline_s'] != start + 7200
|
|
or data['clock']['boot_id'] != b.boot_id()):
|
|
raise ValueError('Worker changed supervisor lifetime')
|
|
physics = result['physics']
|
|
attempted, completed, init, rollout, startup = [physics[k] for k in (
|
|
'attempted_shared_calls', 'completed_shared_calls', 'initialization_attempted_shared_calls',
|
|
'rollout_attempted_shared_calls', 'startup_completed_without_task_audit')]
|
|
if (any(type(x) is not int for x in (attempted, completed, init, rollout, startup))
|
|
or not 0 <= startup <= init <= 1000 or rollout != 67960
|
|
or attempted != completed or completed != init + rollout or completed > 73460):
|
|
raise ValueError('Actual shared physics completion/cap evidence mismatch')
|
|
safety = physics['safety']
|
|
gates = read(ROOT / manifest['training']['safety_config'])['gates']
|
|
values = [safety[k] for k in ('max_effort_velocity_ratio', 'max_joint_excess_rad', 'max_guide_vertical_N')]
|
|
if (safety['physics_samples'] != completed - startup or safety['finite'] is not True
|
|
or any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values)
|
|
or values[0] > gates['normalized_effort_velocity_max']
|
|
or values[1] > gates['joint_limit_excess_rad_max'] or values[2] != 0):
|
|
raise ValueError('Missing/failing global physical safety evidence')
|
|
for label in ('analytic_zero', 'frozen_u256'):
|
|
rows = read(root / ('validation_' + label + '.json'))['rows']
|
|
if (len(rows) != 128 or [r.get('env') for r in rows] != list(range(128))
|
|
or any(r.get('success') is not True or r.get('functional_audit', {}).get('sim_functional_passed')
|
|
is not True for r in rows)):
|
|
raise ValueError('Missing original baseline functional128/128 evidence')
|
|
|
|
|
|
def commit_candidate(root, result, manifest, budget):
|
|
"""Independent validation and sole-authority last atomic commit; no worker lock."""
|
|
# Import CPU protocol only after worker exit/cleanup; never initialize CUDA.
|
|
selection_path = ROOT / owner.PACKAGE / 'legs/contact_cycle_sim/rootxy_selection_v2.py'
|
|
spec = importlib.util.spec_from_file_location('_rootxy_selection_cpu', selection_path)
|
|
selection = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(selection)
|
|
if (result.get('status') != 'completed_provisional' or result.get('training_complete') is not True
|
|
or result.get('accepted_skill') is not False or result.get('checkpoints_selectable') is not False
|
|
or result.get('postclose_identity_verified') is not True):
|
|
raise ValueError('No successful provisional worker result')
|
|
ranked = []
|
|
for update in (64, 128, 192, 256):
|
|
document = read(root / f'validation_u{update:04d}.json')
|
|
rows = document['rows']
|
|
# Recheck original functional audit, not just worker's candidate label.
|
|
if any(r.get('functional_audit', {}).get('sim_functional_passed') is not True for r in rows):
|
|
raise ValueError('Missing original functional128/128 evidence')
|
|
key = selection.candidate_key(rows, update)
|
|
if key is not None:
|
|
ranked.append((key, update))
|
|
candidate = result.get('candidate')
|
|
if not ranked:
|
|
if candidate is not None or result.get('outcome') != 'no_candidate':
|
|
raise ValueError('Ineligible worker candidate')
|
|
return None
|
|
key, update = min(ranked)
|
|
if (not isinstance(candidate, dict) or set(candidate) != {'path', 'sha256', 'update', 'selection_key'}
|
|
or candidate['update'] != update or candidate['path'] != f'checkpoint_u{update:04d}.pt'
|
|
or candidate['selection_key'] != list(key) or b.digest(root / candidate['path']) != candidate['sha256']):
|
|
raise ValueError('Candidate selection/hash mismatch')
|
|
lock = {'format': 'skillfactory.rootxy-supervisor-candidate.v2', 'candidate': candidate,
|
|
'manifest_file_sha256': b.digest(root / 'manifest.json'), 'accepted_skill': False,
|
|
'authority': 'waited_exit_postexit_identity_clean_owned_GPU_cleanup_original_eligibility',
|
|
'downstream_requires_separate_grant': True}
|
|
budget.write(root / 'locked_candidate.json', lock, cleanup=True)
|
|
return lock
|
|
|
|
|
|
def supervise(args):
|
|
start = b.boottime() # includes admission/startup; never reset in worker
|
|
manifest, grant, policy = admission(args, start)
|
|
mode = 'prepare' if args.prepare else 'run'
|
|
if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'):
|
|
raise RuntimeError('pidfd supervisor required')
|
|
owned = owner.OwnedProcesses()
|
|
gpu = GPUOwnership(grant['gpu_identity'])
|
|
environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
|
|
CUDA_VISIBLE_DEVICES=grant['gpu_identity']['uuid'])
|
|
owner.launch_command([], environment)
|
|
if mode == 'prepare':
|
|
b.durable_directory(args.output)
|
|
lock = (args.output / (mode + '-claim.lock')).open('xb')
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
budget = GuardBudget(args.output, start, mode)
|
|
monitor = child = capture = channels = None
|
|
errors, handlers = [], {}
|
|
clean = waited = False
|
|
result = None
|
|
filesystem = None
|
|
postexit = None
|
|
|
|
def cleanup_pump():
|
|
if capture is not None:
|
|
capture.pump()
|
|
if monitor is not None:
|
|
try:
|
|
monitor.check()
|
|
except BaseException as error:
|
|
if not any(e.startswith('native_cleanup:') for e in errors):
|
|
errors.append('native_cleanup:' + str(error)[-2048:])
|
|
|
|
try:
|
|
monitor = owner.NativeMonitor(policy, SimpleNamespace(boottime=b.boottime, STORAGE_SCOPE=STORAGE_SCOPE))
|
|
budget.monitor = monitor
|
|
handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True))
|
|
for s in (signal.SIGTERM, signal.SIGINT)}
|
|
budget.write(args.output / (mode + '-reservation.json'), {
|
|
'profile': PROFILE, 'mode': mode, 'start_boottime': start, 'deadline_boottime': budget.deadline,
|
|
'boot_id': budget.boot, 'grant': grant,
|
|
'reserved_optimization_environment_transitions': 0 if mode == 'prepare' else 1048576,
|
|
'reserved_validation_environment_transitions': 0 if mode == 'prepare' else 806400,
|
|
'reserved_rollout_shared_calls': 0 if mode == 'prepare' else 72460,
|
|
'initialization_shared_calls_cap': 0 if mode == 'prepare' else 1000,
|
|
'no_refund': True, 'accepted_skill': False})
|
|
budget.write(args.output / (mode + '-manifest.json'), manifest)
|
|
if mode == 'prepare':
|
|
for relative, digest in manifest['sources'].items():
|
|
destination = args.output / 'source_snapshot' / relative
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
data = (ROOT / relative).read_bytes()
|
|
budget.check(len(data))
|
|
b.durable_write(destination, data)
|
|
original_manifest = Path(args.manifest).read_bytes()
|
|
budget.check(len(original_manifest))
|
|
b.durable_write(args.output / 'manifest.json', original_manifest)
|
|
filesystem = owner.bound_filesystem_inputs(original_manifest)
|
|
else:
|
|
filesystem = owner.bound_filesystem_inputs((args.output / 'training-binding.json').read_bytes())
|
|
owner.verify_filesystem_inputs(filesystem)
|
|
budget.write(args.output / (mode + '-grant.json'), grant)
|
|
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
|
raise RuntimeError('Subreaper setup failed')
|
|
native = args.output / 'native'
|
|
if native != native.resolve():
|
|
raise ValueError('Native redirect symlink forbidden')
|
|
native.mkdir(exist_ok=True)
|
|
for directory in redirects(args.output, mode).values():
|
|
Path(directory).mkdir(parents=True, exist_ok=False)
|
|
environment.update(redirects(args.output, mode))
|
|
channels = socket.socketpair()
|
|
parent, child_channel = channels
|
|
child_channel.set_inheritable(True)
|
|
budget.check()
|
|
child = subprocess.Popen(owner.launch_command(
|
|
[str(ROOT / SCRIPT), '--_worker-fd', str(child_channel.fileno())], environment),
|
|
cwd=ROOT, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
start_new_session=True, pass_fds=(child_channel.fileno(),))
|
|
try:
|
|
owned.attach(child)
|
|
except BaseException:
|
|
owned.unknown = True
|
|
raise
|
|
child_channel.close()
|
|
capture = owner.StdoutCapture(child.stdout, args.output / (mode + '-stdout.log'), b)
|
|
budget.write(args.output / 'monitor-heartbeat.json', {
|
|
'boot_id': budget.boot, 'boottime': b.boottime(), 'external_growth': budget.external_growth})
|
|
owner.exchange_worker(child, parent, {
|
|
'mode': mode, 'output': str(args.output), 'start': start, 'boot_id': budget.boot,
|
|
'manifest_sha256': args.trusted_manifest_sha256, 'grant': grant,
|
|
'launch_identity': launch_identity()}, budget, capture, owned, monitor, b)
|
|
next_heartbeat = 0
|
|
while child.poll() is None or not capture.eof:
|
|
capture.check()
|
|
budget.check()
|
|
gpu.check(owned)
|
|
if b.boottime() >= next_heartbeat:
|
|
budget.write(args.output / 'monitor-heartbeat.json', {
|
|
'boot_id': budget.boot, 'boottime': b.boottime(), 'external_growth': budget.external_growth})
|
|
next_heartbeat = b.boottime() + .5
|
|
time.sleep(.01)
|
|
child.wait(timeout=max(.01, budget.deadline - b.boottime()))
|
|
waited = True
|
|
result = read(args.output / (mode + '-worker-result.json'))
|
|
if child.returncode != 0:
|
|
raise RuntimeError('Worker nonzero exit')
|
|
except BaseException as error:
|
|
errors.append(str(error)[-8192:])
|
|
finally:
|
|
try:
|
|
if channels:
|
|
for channel in channels:
|
|
channel.close()
|
|
if child is not None:
|
|
if capture is None and child.stdout is not None:
|
|
child.stdout.close()
|
|
try:
|
|
clean = owner.cleanup_group(child, budget.deadline, b, owned, cleanup_pump)
|
|
child.wait(timeout=max(.01, budget.deadline - b.boottime()))
|
|
waited = True
|
|
except BaseException as error:
|
|
errors.append('cleanup:' + str(error)[-2048:])
|
|
if capture is not None:
|
|
try:
|
|
capture.close()
|
|
if capture.discarded or capture.error or not capture.eof:
|
|
raise RuntimeError('Incomplete/overflowed stdout')
|
|
except BaseException as error:
|
|
errors.append(str(error)[-2048:])
|
|
try:
|
|
if not clean or not waited or child is None or child.returncode != 0 or owned.unknown:
|
|
raise RuntimeError('No clean independently waited worker exit')
|
|
gpu.check(owned, final=True)
|
|
verify_sources(manifest)
|
|
if filesystem is None:
|
|
raise RuntimeError('Missing prelaunch filesystem identity')
|
|
postexit = owner.verify_filesystem_inputs(filesystem)
|
|
binding_files = owner.bound_filesystem_inputs((args.output / 'training-binding.json').read_bytes())
|
|
owner.verify_filesystem_inputs(binding_files)
|
|
if result is None or result.get('postclose_identity_verified') is not True:
|
|
raise RuntimeError('Missing postclose same-object verification')
|
|
if mode == 'prepare':
|
|
if result.get('status') != 'prepared_config_only' or result.get('physics_calls') != 0:
|
|
raise RuntimeError('PREP incomplete/config-only violation')
|
|
else:
|
|
if result.get('status') != 'completed_provisional' or result.get('training_complete') is not True:
|
|
raise RuntimeError('Training incomplete')
|
|
validate_completion(args.output, result, manifest, start)
|
|
budget.check(cleanup=True)
|
|
monitor.check(force=True)
|
|
except BaseException as error:
|
|
errors.append(str(error)[-4096:])
|
|
summary = {'success': not errors, 'mode': mode, 'cleanup_clean': clean, 'waited_exit': waited,
|
|
'returncode': None if child is None else child.returncode, 'errors': errors,
|
|
'accepted_skill': False, 'postexit': postexit, 'owned': sorted(owned.known),
|
|
'storage': None if monitor is None else monitor.report()}
|
|
budget.write(args.output / (mode + '-supervisor.json'), summary, cleanup=True)
|
|
if not errors and mode == 'prepare':
|
|
prepared = read(args.output / 'config-preparation-provisional.json')
|
|
prepared.update(supervisor_sha256=b.digest(args.output / 'prepare-supervisor.json'),
|
|
worker_result_sha256=b.digest(args.output / 'prepare-worker-result.json'),
|
|
postexit_verified=True)
|
|
try:
|
|
budget.write(args.output / 'preparation.json', prepared, cleanup=True)
|
|
except BaseException:
|
|
(args.output / 'preparation.json').unlink(missing_ok=True)
|
|
raise
|
|
if not errors and mode == 'run':
|
|
# Last operation capable of committing success. Failure leaves no candidate.
|
|
try:
|
|
commit_candidate(args.output, result, manifest, budget)
|
|
except BaseException as error:
|
|
(args.output / 'locked_candidate.json').unlink(missing_ok=True)
|
|
summary.update(success=False, errors=[*errors, 'candidate_commit:' + str(error)[-2048:]])
|
|
budget.write(args.output / (mode + '-supervisor.json'), summary, cleanup=True)
|
|
raise
|
|
finally:
|
|
for sig, handler in handlers.items():
|
|
signal.signal(sig, handler)
|
|
lock.close()
|
|
return 0 if not errors else 1
|
|
|
|
|
|
def preimport_binding_runtime(binding):
|
|
"""Finite binding-required imports, before config/inventory; no optional skips."""
|
|
for name in dict.fromkeys((*binding.REQUIRED_RUNTIME, *binding.ISAAC_PINS)):
|
|
importlib.import_module(name)
|
|
|
|
|
|
class RuntimeAPI:
|
|
"""Actual Kit recipe; callable only from authenticated worker handoff."""
|
|
def __init__(self, payload, budget, guard):
|
|
self.payload, self.budget, self.guard = payload, budget, guard
|
|
self.output = Path(payload['output'])
|
|
self.mode = payload['mode']
|
|
self.launcher = self.env = self.cfg = self.ledger = None
|
|
self.started = False
|
|
self.physics_hooks = None
|
|
self.manifest = read(self.output / 'manifest.json', payload['manifest_sha256'])
|
|
self.expected = None
|
|
|
|
def start(self):
|
|
from isaaclab.app import AppLauncher
|
|
app_source = self.payload['launch_identity']['app_launcher']
|
|
if (AppLauncher.__module__ != 'isaaclab.app.app_launcher'
|
|
or Path(sys.modules[AppLauncher.__module__].__file__).resolve() != Path(app_source['path'])
|
|
or b.digest(app_source['path']) != app_source['sha256']):
|
|
raise ValueError('Actual installed AppLauncher identity mismatch')
|
|
# Retain launcher even if initialization fails after creating its app.
|
|
self.launcher = AppLauncher.__new__(AppLauncher)
|
|
AppLauncher.__init__(self.launcher, headless=True, device='cuda:0', fast_shutdown=True,
|
|
kit_args=f'--/log/file={self.output / "native" / self.mode / "kit.log"}')
|
|
self.started = True
|
|
for name in ('rootxy_training_v2', 'rootxy_runner_v2', 'rootxy_binding_v2', 'rootxy_runtime_v2'):
|
|
setattr(self, name, importlib.import_module(owner.LEG + name))
|
|
self.p = self.rootxy_training_v2
|
|
self.binding = self.rootxy_binding_v2
|
|
eb = importlib.import_module(owner.LEG + 'evaluation_binding')
|
|
self.eb = eb
|
|
for name in self.binding.ROLES:
|
|
if name != '__main__':
|
|
importlib.import_module(name)
|
|
# Do not depend on an evaluation recorder to trigger lazy backends.
|
|
preimport_binding_runtime(eb)
|
|
# Match the finite installed constructor preimports, never a whole interpreter scan.
|
|
owner.preimport_replicator()
|
|
owner.preimport_constructor_modules(eb)
|
|
self.constructor_export, self.constructor_metadata = owner.physx_constructor_export(eb, resolve=True)
|
|
self.cfg_class = importlib.import_module(owner.TASK + 'rootxy_cfg_v2').RootXYContactCycleEnvCfg
|
|
self.environment_class = importlib.import_module(owner.TASK + 'env').SimContactCycleEnv
|
|
|
|
def configure(self):
|
|
self.cfg = self.cfg_class()
|
|
self.cfg.scene.num_envs = 128
|
|
self.cfg.seed = 92511
|
|
self.cfg.sim.device = 'cuda:0'
|
|
# This helper only verifies installed stepping and explicitly forwards
|
|
# fixed constructor video config fields; no evaluation environment created.
|
|
owner.preforward_video_recorder(self.cfg, importlib.import_module(owner.TASK + 'evaluation_env'))
|
|
owner.preimport_config_references(self.cfg, self.eb)
|
|
self.binding.normalize_config(self.cfg) # exact128-env PREconstruction only
|
|
import torch
|
|
torch.cuda.init()
|
|
torch.cuda.get_rng_state(torch.device('cuda:0'))
|
|
if self.mode == 'prepare':
|
|
self.inventory = self.binding.propose_inventory(ROOT, self.cfg, owner.proposed_native_layouts)
|
|
self.expected = self.binding.snapshot(ROOT, self.output, self.cfg, self.inventory, self.manifest)
|
|
else:
|
|
self.expected = read(self.output / 'training-binding.json', self.payload['grant']['binding_sha256'])
|
|
self.inventory = self.expected['dependency_inventory']
|
|
if self.mode == 'run':
|
|
outer = self
|
|
class Ledger(self.p.RootXYBudgetLedger):
|
|
def guard(self, reserve_bytes=0):
|
|
outer.guard()
|
|
outer.budget.check(reserve_bytes)
|
|
return super().guard(reserve_bytes=reserve_bytes)
|
|
self.ledger = Ledger(self.output, self.manifest, launch_start_s=self.payload['start'])
|
|
self.ledger.__enter__()
|
|
|
|
@contextmanager
|
|
def physics(self):
|
|
from isaaclab.sim import SimulationContext
|
|
with self.rootxy_runtime_v2.RootXYPhysicsHooks(
|
|
SimulationContext, self.environment_class, self.guard,
|
|
lambda value: self.budget.write(self.output / 'physics-counts.json', value),
|
|
preparation=self.mode == 'prepare') as hooks:
|
|
# Config construction/normalization itself is under zero-physics PREP hooks.
|
|
self.configure()
|
|
yield hooks
|
|
|
|
def verify(self):
|
|
self.budget.check(cleanup=True)
|
|
verify_sources(self.manifest)
|
|
if self.cfg is None or self.expected is None:
|
|
raise RuntimeError('Startup did not establish actual config identity')
|
|
target, metadata = owner.physx_constructor_export(self.eb, expected=self.constructor_export)
|
|
if metadata != self.constructor_metadata:
|
|
raise ValueError('Constructor export drift')
|
|
self.binding.verify(self.expected, root=ROOT, output=self.output, cfg=self.cfg,
|
|
inventory=self.inventory, manifest=self.manifest)
|
|
if self.env is not None and getattr(self.env, 'cfg', None) is not self.cfg:
|
|
raise ValueError('Constructor substituted actual config object')
|
|
|
|
def prepare(self):
|
|
self.budget.write(self.output / 'training-binding.json', self.expected)
|
|
result = {'format': PREPARATION_FORMAT, 'status': 'prepared_config_only',
|
|
'manifest_sha256': self.payload['manifest_sha256'],
|
|
'binding_sha256': b.digest(self.output / 'training-binding.json'),
|
|
'artifact_root': str(self.output), 'physics_calls': 0, 'num_envs': 128,
|
|
'runtime_qualified': False, 'p2_decision': 'pending', 'accepted_skill': False,
|
|
'recipe': {'device': 'cuda:0', 'train_seed': 92511, 'development_seed': 93511,
|
|
'environment_constructed': False, 'inference_executed': False,
|
|
'profile': PROFILE, 'hooks': self.binding.HOOKS},
|
|
'ready_gaps': ['parent review of actual config-only binding',
|
|
'separate approved P2 decision and fresh run grant',
|
|
'immediate postconstructor actual-used identity and reset safety qualification']}
|
|
self.budget.write(self.output / 'config-preparation-provisional.json', result)
|
|
return result
|
|
|
|
def construct(self):
|
|
self.guard()
|
|
self.env = self.environment_class.__new__(self.environment_class)
|
|
self.environment_class.__init__(self.env, cfg=self.cfg)
|
|
|
|
def check_actual(self):
|
|
importlib.import_module(owner.TASK + 'rootxy_cfg_v2').verify_actual_config(self.env)
|
|
if self.env._physics_handles_decimation:
|
|
raise RuntimeError('Require scalar 500Hz physics stepping')
|
|
|
|
def safety(self):
|
|
from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit
|
|
gates = read(ROOT / self.p.PILOT['safety_config'])['gates']
|
|
return PhysicalSafetyAudit(self.env, gates)
|
|
|
|
def learn(self):
|
|
import torch
|
|
from audit_contact_cycle_functional import evaluate
|
|
trace_path = ROOT / self.p.replay.CANDIDATE
|
|
trace_path = trace_path.parent / 'validation_u0256_trace.pt'
|
|
if b.digest(trace_path) != self.manifest['parity_trace']['sha256']:
|
|
raise ValueError('Saved CPU parity trace drift')
|
|
trace = torch.load(trace_path, map_location='cpu', weights_only=True)
|
|
observations = next(iter(trace.values()))['actor_input'][::100, 0].contiguous()
|
|
del trace
|
|
|
|
def guard():
|
|
self.guard()
|
|
self.physics_hooks.safety_boundary()
|
|
|
|
def before_control(kind):
|
|
if kind not in ('optimization', 'validation'):
|
|
raise ValueError('Unknown control budget kind')
|
|
guard()
|
|
self.physics_hooks.mode = 'rollout'
|
|
|
|
def after_control(kind):
|
|
self.physics_hooks.mode = 'initialization'
|
|
guard()
|
|
|
|
hooks = self.rootxy_runner_v2.RuntimeHooks(
|
|
admit=lambda manifest: {'manifest_sha256': self.p.digest(manifest), 'p2_review_approved': True,
|
|
'sole_cleanup_owner': True, 'native_monitor_active': True},
|
|
guard=guard, verify_identity=self.verify, before_control=before_control, after_control=after_control,
|
|
progress=lambda value: guard(), finish=lambda value: guard())
|
|
return self.rootxy_runner_v2.execute(self.env, ROOT, self.output, self.manifest, self.ledger,
|
|
observations, hooks, evaluate)
|
|
|
|
def close_env(self):
|
|
if self.env is not None:
|
|
self.env.close()
|
|
|
|
def close_app(self, code):
|
|
error = None
|
|
try:
|
|
if self.ledger is not None:
|
|
self.ledger.__exit__()
|
|
except BaseException as caught:
|
|
error = caught
|
|
app = None if self.launcher is None else getattr(self.launcher, '_app', None)
|
|
if app is not None:
|
|
# Fast shutdown may exit instead of returning: never pass zero after
|
|
# another teardown failure and rely on later Python unwinding.
|
|
app.close(exit_code=1 if error is not None else code)
|
|
if error is not None:
|
|
raise error
|
|
|
|
|
|
def run_worker(fd):
|
|
payload = worker_payload(fd) # no unauthenticated dictionary entrypoint
|
|
if payload['mode'] not in ('prepare', 'run') or payload['boot_id'] != b.boot_id():
|
|
raise PermissionError('Invalid authenticated RootXY handoff')
|
|
root = checked_root(payload['output'])
|
|
reservation = read(root / (payload['mode'] + '-reservation.json'))
|
|
if (reservation['grant'] != payload['grant'] or reservation['start_boottime'] != payload['start']
|
|
or reservation['boot_id'] != payload['boot_id']):
|
|
raise PermissionError('RootXY reservation mismatch')
|
|
budget = GuardBudget(root, payload['start'], payload['mode'])
|
|
|
|
def guard():
|
|
heartbeat = read(root / 'monitor-heartbeat.json')
|
|
if (heartbeat['boot_id'] != b.boot_id()
|
|
or not 0 <= b.boottime() - heartbeat['boottime'] <= 5):
|
|
raise RuntimeError('Native monitor heartbeat missing/stale')
|
|
budget.external_growth = heartbeat['external_growth']
|
|
budget.check()
|
|
|
|
handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True))
|
|
for s in (signal.SIGTERM, signal.SIGINT)}
|
|
# Pure hook module is loaded under a private identity before Kit, not a fake
|
|
# SkillFactory package. Actual runtime imports after AppLauncher use real paths.
|
|
spec = importlib.util.spec_from_file_location('_rootxy_lifecycle_cpu', ROOT / owner.PACKAGE /
|
|
'legs/contact_cycle_sim/rootxy_runtime_v2.py')
|
|
runtime = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(runtime)
|
|
api = RuntimeAPI(payload, budget, guard)
|
|
verify_sources(api.manifest)
|
|
try:
|
|
runtime.worker_lifecycle(api, prepare=payload['mode'] == 'prepare', guard=guard, verify=api.verify,
|
|
publish=lambda value: budget.write(
|
|
root / (payload['mode'] + '-worker-result.json'), value, cleanup=True))
|
|
return 0
|
|
finally:
|
|
for sig, handler in handlers.items():
|
|
signal.signal(sig, handler)
|