9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
311 lines
14 KiB
Python
311 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Dedicated ONE preparation-only supervisor; never an execution grant or retry.
|
|
|
|
Native scans run on a daemon thread, never on the BOOTTIME deadline owner.
|
|
The main thread alone spawns, discovers, signals and reaps the Kit process tree.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import ctypes
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SPEC = importlib.util.spec_from_file_location('preparation_runner', ROOT / 'scripts/evaluate_cycle_sim_residual.py')
|
|
r = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(r)
|
|
b = r.cpu_module('evaluation_budget')
|
|
TOTAL_SECONDS = 600
|
|
PREPARE_SECONDS = 480
|
|
CANDIDATE = ROOT / 'logs/contact_cycle/sim_residual_training_v1/prepared01/candidate_u0256.pt'
|
|
|
|
|
|
class AsyncNativeMonitor:
|
|
"""A blocked native scan cannot block the sole deadline/cleanup owner."""
|
|
def __init__(self, policy):
|
|
self.state = (None, None, None)
|
|
self.stop = threading.Event()
|
|
self.policy = policy
|
|
self.thread = threading.Thread(target=self._scan, daemon=True, name='native-monitor')
|
|
self.thread.start()
|
|
|
|
def _scan(self):
|
|
monitor = None
|
|
try:
|
|
monitor = r.NativeMonitor(self.policy, b)
|
|
self.state = (b.boottime(), monitor.report(), None)
|
|
while not self.stop.wait(self.policy['check_interval_seconds']):
|
|
monitor.check(force=True)
|
|
self.state = (b.boottime(), monitor.report(), None)
|
|
except BaseException as error:
|
|
completed, report, _ = self.state
|
|
report = getattr(error, 'report', monitor.report() if monitor is not None else report)
|
|
self.state = (completed, report, repr(error))
|
|
|
|
def check(self):
|
|
completed, report, error = self.state
|
|
if error:
|
|
raise RuntimeError('preparation_native_monitor: ' + error)
|
|
if completed is not None and b.boottime() - completed > 2:
|
|
raise RuntimeError('preparation_native_scan_stale_over_2_seconds')
|
|
return report
|
|
|
|
|
|
def write(root, name, value, *, final=False):
|
|
payload = b.encode(value)
|
|
reserve = 0 if final else b.HEADROOM_BYTES + b.STDOUT_ALLOWANCE
|
|
if b.artifact_size(root, exclude_native=True) + len(payload) > b.CAP_BYTES - reserve:
|
|
raise RuntimeError('preparation_aggregate_output_cap')
|
|
b.durable_write(root / name, payload)
|
|
|
|
|
|
def without_core_dumps(command):
|
|
"""Child-only soft/hard RLIMIT_CORE=0; neither a crash fix nor global policy.
|
|
|
|
Use an exec shell rather than unsafe preexec_fn in this threaded supervisor.
|
|
Piped OS crash handlers may ignore RLIMIT_CORE; old evidence is untouched.
|
|
"""
|
|
return ['/usr/bin/bash', '-c',
|
|
'ulimit -S -c 0 && ulimit -H -c 0 || exit 125; exec "$@"',
|
|
'preparation-no-core', *command]
|
|
|
|
|
|
def postexit_identity(root, frozen_before, protocol=r.PROTOCOLS[0]):
|
|
"""External owner: require durable child evidence, independently rehash files."""
|
|
output = root / preparation_name(protocol)
|
|
manifest = (output / 'binding.json').read_bytes()
|
|
report = json.loads((output / 'preparation.json').read_bytes())
|
|
if (report.get('preclose_identity_verified') is not True
|
|
or report.get('manifest_sha256_for_independent_review') != r.hashlib.sha256(manifest).hexdigest()
|
|
or (output / 'preparation-failure.json').exists()):
|
|
raise ValueError('preparation_missing_or_failed_preclose_identity')
|
|
evidence = json.loads((output / 'preclose-filesystem.json').read_bytes())
|
|
independent = r.bound_filesystem_inputs(manifest)
|
|
independent['files'].update(frozen_before)
|
|
if evidence != independent:
|
|
raise ValueError('preparation_postexit_bound_inputs_changed')
|
|
return r.verify_filesystem_inputs(evidence)
|
|
|
|
|
|
def collect_postexit_identity(root, frozen_before, child, errors, deadline, protocol=r.PROTOCOLS[0]):
|
|
if child is None:
|
|
return None
|
|
result = []
|
|
|
|
def rehash():
|
|
try:
|
|
result.append((postexit_identity(root, frozen_before, protocol), None))
|
|
except BaseException as error:
|
|
result.append((None, repr(error)))
|
|
|
|
# File I/O must not take deadline ownership away from the sole supervisor.
|
|
thread = threading.Thread(target=rehash, daemon=True, name='postexit-identity')
|
|
thread.start()
|
|
while thread.is_alive():
|
|
remaining = deadline - b.boottime()
|
|
if remaining <= 0:
|
|
errors.append('preparation_postexit_identity_deadline')
|
|
return None
|
|
thread.join(min(.05, remaining))
|
|
report, error = result[0]
|
|
if error:
|
|
errors.append(error)
|
|
return report
|
|
|
|
|
|
def preparation_name(protocol):
|
|
return 'prep-' + r.protocol_stage(protocol).lower()
|
|
|
|
|
|
def supervise(root, policy_path, trusted_policy_digest, protocol=r.PROTOCOLS[0]):
|
|
name = preparation_name(protocol)
|
|
if r.resource_profile(protocol) is not None:
|
|
b.extended_historical_calls(root) # same finite root allowlist as reservation
|
|
started = b.boottime()
|
|
deadline = started + TOTAL_SECONDS
|
|
if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'):
|
|
raise RuntimeError('preparation_pidfd_required')
|
|
r.protect_artifact_path(root)
|
|
if any((root / name).exists() for name in ('prep-p1', 'prep-p2', 'native/prep-p1', 'native/prep-p2', 'preparation-start.json', 'budget.json')):
|
|
raise RuntimeError('preparation_requires_fresh_campaign_no_retry')
|
|
policy = r.storage_policy_check(SimpleNamespace(
|
|
artifact_root=root, protocol=protocol, storage_policy=policy_path,
|
|
trusted_storage_policy_sha256=trusted_policy_digest), b)
|
|
if policy['not_after_boottime'] < deadline:
|
|
raise RuntimeError('preparation_policy_expiry_before_cleanup')
|
|
owned = r.OwnedProcesses()
|
|
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
|
raise RuntimeError('preparation_subreaper_setup_failed')
|
|
before = b.digest(CANDIDATE)
|
|
frozen_before = {str(p): b.digest(p) for p in CANDIDATE.parent.rglob('*') if p.is_file()}
|
|
write(root, 'preparation-start.json', {
|
|
'started_boottime': started, 'deadline_boottime': deadline,
|
|
'prepare_cutoff_boottime': started + PREPARE_SECONDS,
|
|
'supervisor_pid_starttime': owned.supervisor, 'dedicated_sole_spawner': True,
|
|
'candidate_before_sha256': before, 'kit_only_attempt': True,
|
|
'actual_run_reservations_consumed': 0, 'execution_authorized': False,
|
|
'storage_policy_sha256': trusted_policy_digest, 'policy_role': 'preparation-monitor-only',
|
|
'supervisor_sha256': b.digest(Path(__file__)), 'runner_sha256': b.digest(ROOT / r.SCRIPT),
|
|
'independent_native_scan_deadline_owner': True,
|
|
'stdout_limit_bytes': b.STDOUT_BYTES, 'cleanup_reserve_bytes': b.HEADROOM_BYTES,
|
|
'stdout_allowance_bytes': b.STDOUT_ALLOWANCE,
|
|
'child_rlimit_core': {'soft': 0, 'hard': 0, 'scope': 'launched child and descendants only'},
|
|
})
|
|
monitor = AsyncNativeMonitor(policy)
|
|
child = capture = None
|
|
command = None
|
|
clean = False
|
|
errors = []
|
|
interrupted = threading.Event()
|
|
handlers = {sig: signal.signal(sig, lambda *_: interrupted.set()) for sig in (signal.SIGTERM, signal.SIGINT)}
|
|
native_report = None
|
|
launch_started = launch_ended = None
|
|
|
|
def check():
|
|
nonlocal native_report
|
|
if interrupted.is_set():
|
|
raise RuntimeError('preparation_interrupted')
|
|
if b.boottime() >= started + PREPARE_SECONDS:
|
|
raise RuntimeError('preparation_480s_deadline')
|
|
native_report = monitor.check()
|
|
if b.artifact_size(root, exclude_native=True) > b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE:
|
|
raise RuntimeError('preparation_aggregate_output_cap')
|
|
|
|
def cleanup_pump():
|
|
nonlocal native_report
|
|
if capture is not None:
|
|
try:
|
|
capture.pump()
|
|
except Exception as error:
|
|
text = repr(error)
|
|
if text not in errors:
|
|
errors.append(text)
|
|
try:
|
|
native_report = monitor.check()
|
|
except Exception as error:
|
|
text = repr(error)
|
|
if text not in errors:
|
|
errors.append(text)
|
|
|
|
try:
|
|
while native_report is None:
|
|
check()
|
|
time.sleep(.05)
|
|
write(root, 'native-baseline.json', native_report)
|
|
native = root / 'native'
|
|
if not native.exists():
|
|
b.durable_directory(native)
|
|
support = native / 'preparation-support'
|
|
b.durable_directory(support)
|
|
redirects = r.native_redirects(support, preparation=True)
|
|
for directory in redirects.values():
|
|
b.durable_directory(Path(directory))
|
|
environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
|
|
**redirects)
|
|
for key in ('CONDA_PREFIX', 'VIRTUAL_ENV', 'PYTHONEXE', 'PYTHONHOME', 'args'):
|
|
environment.pop(key, None)
|
|
environment['PYTHONPATH'] = ':'.join([
|
|
str(ROOT / 'source/SkillFactory'),
|
|
*['/home/cen/IsaacLab/source/' + name for name in
|
|
('isaaclab', 'isaaclab_physx', 'isaaclab_assets', 'isaaclab_tasks')]])
|
|
arguments = [str(ROOT / r.SCRIPT), '--prepare-binding', '--discover-inventory',
|
|
'--protocol', protocol, '--trial-index', '0', '--device', 'cuda:0',
|
|
'--artifact-root', str(root), '--output', str(root / name)]
|
|
command = without_core_dumps(r.launch_command(arguments, environment))
|
|
write(root, 'preparation-command.json', {'argv': command, 'cwd': str(ROOT),
|
|
'environment_overrides': {k: environment[k] for k in ('HOME', 'TMPDIR', 'XDG_CACHE_HOME',
|
|
'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'PYTHONPATH', 'PYTHONDONTWRITEBYTECODE',
|
|
'PYTEST_DISABLE_PLUGIN_AUTOLOAD')}, 'launch_identity': r.launch_identity()})
|
|
check()
|
|
launch_started = b.boottime()
|
|
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, start_new_session=True)
|
|
try:
|
|
owned.attach(child)
|
|
except BaseException:
|
|
owned.unknown = True
|
|
raise
|
|
capture = r.StdoutCapture(child.stdout, root / 'preparation.stdout.log', b)
|
|
while not capture.eof or child.poll() is None:
|
|
capture.check()
|
|
check()
|
|
owned.discover()
|
|
time.sleep(.02)
|
|
launch_ended = b.boottime()
|
|
if child.returncode != 0:
|
|
raise RuntimeError('preparation_child_exit_' + str(child.returncode))
|
|
except BaseException as error:
|
|
errors.append(repr(error))
|
|
finally:
|
|
try:
|
|
if child is not None:
|
|
try:
|
|
if capture is None and child.stdout is not None:
|
|
child.stdout.close()
|
|
finally:
|
|
clean = r.cleanup_group(child, deadline, b, owned, cleanup_pump)
|
|
else:
|
|
clean = not owned.discover() and not owned.unknown
|
|
if capture is not None:
|
|
capture.close()
|
|
if capture.discarded or capture.error:
|
|
errors.append('preparation_stdout_incomplete')
|
|
native_report = monitor.check()
|
|
except BaseException as error:
|
|
errors.append(repr(error))
|
|
finally:
|
|
monitor.stop.set() # never join a blocked filesystem scanner
|
|
for sig, handler in handlers.items():
|
|
signal.signal(sig, handler)
|
|
postexit = collect_postexit_identity(root, frozen_before, child, errors, deadline, protocol)
|
|
after = b.digest(CANDIDATE)
|
|
if after != before:
|
|
errors.append('candidate_changed')
|
|
remaining = owned.discover()
|
|
report = {
|
|
'execution_authorized': False, 'kit_only_launch_started': child is not None,
|
|
'actual_run_reservations_consumed': 0, 'command_record': 'preparation-command.json' if command else None,
|
|
'started_boottime': started, 'finished_boottime': b.boottime(), 'deadline_boottime': deadline,
|
|
'launch_started_boottime': launch_started, 'launch_ended_boottime': launch_ended,
|
|
'returncode': child.returncode if child is not None else None,
|
|
'postexit_identity': postexit,
|
|
'identity_ownership': 'child actual cfg preclose; parent immutable files and process cleanup postexit',
|
|
'cleanup_clean': clean, 'ownership_unknown': owned.unknown,
|
|
'owned_remaining': remaining, 'owned_identities': sorted(owned.known),
|
|
'stdout_bytes': capture.written if capture else 0, 'stdout_discarded': capture.discarded if capture else 0,
|
|
'stdout_eof': capture.eof if capture else False, 'errors': errors,
|
|
'monitor': monitor.state[1], 'monitor_last_completed_boottime': monitor.state[0],
|
|
'monitor_thread_still_alive': monitor.thread.is_alive(),
|
|
'candidate_before_sha256': before, 'candidate_after_sha256': after,
|
|
'protocol': protocol, 'preparation_directory': name,
|
|
'outcomes': {filename: (root / name / filename).is_file()
|
|
for filename in ('inventory.proposed.json', 'binding.json', 'preparation.json')},
|
|
'artifact_bytes_before_report': b.artifact_size(root, exclude_native=True),
|
|
'scope': b.STORAGE_SCOPE, 'native_hard_cap': False,
|
|
'inventory_and_manifest_trust': 'unreviewed proposals only; no execution identity approved',
|
|
}
|
|
write(root, 'preparation-supervisor.json', report, final=True)
|
|
return 0 if child is not None and clean and not remaining and not errors else 1
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--protocol', choices=r.PROTOCOLS, default=r.PROTOCOLS[0])
|
|
parser.add_argument('--artifact-root', type=Path, required=True)
|
|
parser.add_argument('--storage-policy', type=Path, required=True)
|
|
parser.add_argument('--trusted-storage-policy-sha256', required=True)
|
|
args = parser.parse_args(argv)
|
|
return supervise(args.artifact_root, args.storage_policy, args.trusted_storage_policy_sha256, args.protocol)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|