#!/usr/bin/env python3 """Dedicated ONE preparation-only supervisor; never an execution grant or retry. Native scans run on a daemon thread, never on the BOOTTIME deadline owner. The main thread alone spawns, discovers, signals and reaps the Kit process tree. """ from __future__ import annotations import argparse import ctypes import importlib.util import json import os import signal import subprocess import threading import time from pathlib import Path from types import SimpleNamespace ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location('preparation_runner', ROOT / 'scripts/evaluate_cycle_sim_residual.py') r = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(r) b = r.cpu_module('evaluation_budget') TOTAL_SECONDS = 600 PREPARE_SECONDS = 480 CANDIDATE = ROOT / 'logs/contact_cycle/sim_residual_training_v1/prepared01/candidate_u0256.pt' class AsyncNativeMonitor: """A blocked native scan cannot block the sole deadline/cleanup owner.""" def __init__(self, policy): self.state = (None, None, None) self.stop = threading.Event() self.policy = policy self.thread = threading.Thread(target=self._scan, daemon=True, name='native-monitor') self.thread.start() def _scan(self): monitor = None try: monitor = r.NativeMonitor(self.policy, b) self.state = (b.boottime(), monitor.report(), None) while not self.stop.wait(self.policy['check_interval_seconds']): monitor.check(force=True) self.state = (b.boottime(), monitor.report(), None) except BaseException as error: completed, report, _ = self.state report = getattr(error, 'report', monitor.report() if monitor is not None else report) self.state = (completed, report, repr(error)) def check(self): completed, report, error = self.state if error: raise RuntimeError('preparation_native_monitor: ' + error) if completed is not None and b.boottime() - completed > 2: raise RuntimeError('preparation_native_scan_stale_over_2_seconds') return report def write(root, name, value, *, final=False): payload = b.encode(value) reserve = 0 if final else b.HEADROOM_BYTES + b.STDOUT_ALLOWANCE if b.artifact_size(root, exclude_native=True) + len(payload) > b.CAP_BYTES - reserve: raise RuntimeError('preparation_aggregate_output_cap') b.durable_write(root / name, payload) def without_core_dumps(command): """Child-only soft/hard RLIMIT_CORE=0; neither a crash fix nor global policy. Use an exec shell rather than unsafe preexec_fn in this threaded supervisor. Piped OS crash handlers may ignore RLIMIT_CORE; old evidence is untouched. """ return ['/usr/bin/bash', '-c', 'ulimit -S -c 0 && ulimit -H -c 0 || exit 125; exec "$@"', 'preparation-no-core', *command] def postexit_identity(root, frozen_before, protocol=r.PROTOCOLS[0]): """External owner: require durable child evidence, independently rehash files.""" output = root / preparation_name(protocol) manifest = (output / 'binding.json').read_bytes() report = json.loads((output / 'preparation.json').read_bytes()) if (report.get('preclose_identity_verified') is not True or report.get('manifest_sha256_for_independent_review') != r.hashlib.sha256(manifest).hexdigest() or (output / 'preparation-failure.json').exists()): raise ValueError('preparation_missing_or_failed_preclose_identity') evidence = json.loads((output / 'preclose-filesystem.json').read_bytes()) independent = r.bound_filesystem_inputs(manifest) independent['files'].update(frozen_before) if evidence != independent: raise ValueError('preparation_postexit_bound_inputs_changed') return r.verify_filesystem_inputs(evidence) def collect_postexit_identity(root, frozen_before, child, errors, deadline, protocol=r.PROTOCOLS[0]): if child is None: return None result = [] def rehash(): try: result.append((postexit_identity(root, frozen_before, protocol), None)) except BaseException as error: result.append((None, repr(error))) # File I/O must not take deadline ownership away from the sole supervisor. thread = threading.Thread(target=rehash, daemon=True, name='postexit-identity') thread.start() while thread.is_alive(): remaining = deadline - b.boottime() if remaining <= 0: errors.append('preparation_postexit_identity_deadline') return None thread.join(min(.05, remaining)) report, error = result[0] if error: errors.append(error) return report def preparation_name(protocol): return 'prep-' + r.protocol_stage(protocol).lower() def supervise(root, policy_path, trusted_policy_digest, protocol=r.PROTOCOLS[0]): name = preparation_name(protocol) if r.resource_profile(protocol) is not None: b.extended_historical_calls(root) # same finite root allowlist as reservation started = b.boottime() deadline = started + TOTAL_SECONDS if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'): raise RuntimeError('preparation_pidfd_required') r.protect_artifact_path(root) if any((root / name).exists() for name in ('prep-p1', 'prep-p2', 'native/prep-p1', 'native/prep-p2', 'preparation-start.json', 'budget.json')): raise RuntimeError('preparation_requires_fresh_campaign_no_retry') policy = r.storage_policy_check(SimpleNamespace( artifact_root=root, protocol=protocol, storage_policy=policy_path, trusted_storage_policy_sha256=trusted_policy_digest), b) if policy['not_after_boottime'] < deadline: raise RuntimeError('preparation_policy_expiry_before_cleanup') owned = r.OwnedProcesses() if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: raise RuntimeError('preparation_subreaper_setup_failed') before = b.digest(CANDIDATE) frozen_before = {str(p): b.digest(p) for p in CANDIDATE.parent.rglob('*') if p.is_file()} write(root, 'preparation-start.json', { 'started_boottime': started, 'deadline_boottime': deadline, 'prepare_cutoff_boottime': started + PREPARE_SECONDS, 'supervisor_pid_starttime': owned.supervisor, 'dedicated_sole_spawner': True, 'candidate_before_sha256': before, 'kit_only_attempt': True, 'actual_run_reservations_consumed': 0, 'execution_authorized': False, 'storage_policy_sha256': trusted_policy_digest, 'policy_role': 'preparation-monitor-only', 'supervisor_sha256': b.digest(Path(__file__)), 'runner_sha256': b.digest(ROOT / r.SCRIPT), 'independent_native_scan_deadline_owner': True, 'stdout_limit_bytes': b.STDOUT_BYTES, 'cleanup_reserve_bytes': b.HEADROOM_BYTES, 'stdout_allowance_bytes': b.STDOUT_ALLOWANCE, 'child_rlimit_core': {'soft': 0, 'hard': 0, 'scope': 'launched child and descendants only'}, }) monitor = AsyncNativeMonitor(policy) child = capture = None command = None clean = False errors = [] interrupted = threading.Event() handlers = {sig: signal.signal(sig, lambda *_: interrupted.set()) for sig in (signal.SIGTERM, signal.SIGINT)} native_report = None launch_started = launch_ended = None def check(): nonlocal native_report if interrupted.is_set(): raise RuntimeError('preparation_interrupted') if b.boottime() >= started + PREPARE_SECONDS: raise RuntimeError('preparation_480s_deadline') native_report = monitor.check() if b.artifact_size(root, exclude_native=True) > b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE: raise RuntimeError('preparation_aggregate_output_cap') def cleanup_pump(): nonlocal native_report if capture is not None: try: capture.pump() except Exception as error: text = repr(error) if text not in errors: errors.append(text) try: native_report = monitor.check() except Exception as error: text = repr(error) if text not in errors: errors.append(text) try: while native_report is None: check() time.sleep(.05) write(root, 'native-baseline.json', native_report) native = root / 'native' if not native.exists(): b.durable_directory(native) support = native / 'preparation-support' b.durable_directory(support) redirects = r.native_redirects(support, preparation=True) for directory in redirects.values(): b.durable_directory(Path(directory)) environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', **redirects) for key in ('CONDA_PREFIX', 'VIRTUAL_ENV', 'PYTHONEXE', 'PYTHONHOME', 'args'): environment.pop(key, None) environment['PYTHONPATH'] = ':'.join([ str(ROOT / 'source/SkillFactory'), *['/home/cen/IsaacLab/source/' + name for name in ('isaaclab', 'isaaclab_physx', 'isaaclab_assets', 'isaaclab_tasks')]]) arguments = [str(ROOT / r.SCRIPT), '--prepare-binding', '--discover-inventory', '--protocol', protocol, '--trial-index', '0', '--device', 'cuda:0', '--artifact-root', str(root), '--output', str(root / name)] command = without_core_dumps(r.launch_command(arguments, environment)) write(root, 'preparation-command.json', {'argv': command, 'cwd': str(ROOT), 'environment_overrides': {k: environment[k] for k in ('HOME', 'TMPDIR', 'XDG_CACHE_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'PYTHONPATH', 'PYTHONDONTWRITEBYTECODE', 'PYTEST_DISABLE_PLUGIN_AUTOLOAD')}, 'launch_identity': r.launch_identity()}) check() launch_started = b.boottime() child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True) try: owned.attach(child) except BaseException: owned.unknown = True raise capture = r.StdoutCapture(child.stdout, root / 'preparation.stdout.log', b) while not capture.eof or child.poll() is None: capture.check() check() owned.discover() time.sleep(.02) launch_ended = b.boottime() if child.returncode != 0: raise RuntimeError('preparation_child_exit_' + str(child.returncode)) except BaseException as error: errors.append(repr(error)) finally: try: if child is not None: try: if capture is None and child.stdout is not None: child.stdout.close() finally: clean = r.cleanup_group(child, deadline, b, owned, cleanup_pump) else: clean = not owned.discover() and not owned.unknown if capture is not None: capture.close() if capture.discarded or capture.error: errors.append('preparation_stdout_incomplete') native_report = monitor.check() except BaseException as error: errors.append(repr(error)) finally: monitor.stop.set() # never join a blocked filesystem scanner for sig, handler in handlers.items(): signal.signal(sig, handler) postexit = collect_postexit_identity(root, frozen_before, child, errors, deadline, protocol) after = b.digest(CANDIDATE) if after != before: errors.append('candidate_changed') remaining = owned.discover() report = { 'execution_authorized': False, 'kit_only_launch_started': child is not None, 'actual_run_reservations_consumed': 0, 'command_record': 'preparation-command.json' if command else None, 'started_boottime': started, 'finished_boottime': b.boottime(), 'deadline_boottime': deadline, 'launch_started_boottime': launch_started, 'launch_ended_boottime': launch_ended, 'returncode': child.returncode if child is not None else None, 'postexit_identity': postexit, 'identity_ownership': 'child actual cfg preclose; parent immutable files and process cleanup postexit', 'cleanup_clean': clean, 'ownership_unknown': owned.unknown, 'owned_remaining': remaining, 'owned_identities': sorted(owned.known), 'stdout_bytes': capture.written if capture else 0, 'stdout_discarded': capture.discarded if capture else 0, 'stdout_eof': capture.eof if capture else False, 'errors': errors, 'monitor': monitor.state[1], 'monitor_last_completed_boottime': monitor.state[0], 'monitor_thread_still_alive': monitor.thread.is_alive(), 'candidate_before_sha256': before, 'candidate_after_sha256': after, 'protocol': protocol, 'preparation_directory': name, 'outcomes': {filename: (root / name / filename).is_file() for filename in ('inventory.proposed.json', 'binding.json', 'preparation.json')}, 'artifact_bytes_before_report': b.artifact_size(root, exclude_native=True), 'scope': b.STORAGE_SCOPE, 'native_hard_cap': False, 'inventory_and_manifest_trust': 'unreviewed proposals only; no execution identity approved', } write(root, 'preparation-supervisor.json', report, final=True) return 0 if child is not None and clean and not remaining and not errors else 1 def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--protocol', choices=r.PROTOCOLS, default=r.PROTOCOLS[0]) parser.add_argument('--artifact-root', type=Path, required=True) parser.add_argument('--storage-policy', type=Path, required=True) parser.add_argument('--trusted-storage-policy-sha256', required=True) args = parser.parse_args(argv) return supervise(args.artifact_root, args.storage_policy, args.trusted_storage_policy_sha256, args.protocol) if __name__ == '__main__': raise SystemExit(main())