feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
This commit is contained in:
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dedicated ONE preparation-only supervisor; never an execution grant or retry.
|
||||
|
||||
Native scans run on a daemon thread, never on the BOOTTIME deadline owner.
|
||||
The main thread alone spawns, discovers, signals and reaps the Kit process tree.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SPEC = importlib.util.spec_from_file_location('preparation_runner', ROOT / 'scripts/evaluate_cycle_sim_residual.py')
|
||||
r = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(r)
|
||||
b = r.cpu_module('evaluation_budget')
|
||||
TOTAL_SECONDS = 600
|
||||
PREPARE_SECONDS = 480
|
||||
CANDIDATE = ROOT / 'logs/contact_cycle/sim_residual_training_v1/prepared01/candidate_u0256.pt'
|
||||
|
||||
|
||||
class AsyncNativeMonitor:
|
||||
"""A blocked native scan cannot block the sole deadline/cleanup owner."""
|
||||
def __init__(self, policy):
|
||||
self.state = (None, None, None)
|
||||
self.stop = threading.Event()
|
||||
self.policy = policy
|
||||
self.thread = threading.Thread(target=self._scan, daemon=True, name='native-monitor')
|
||||
self.thread.start()
|
||||
|
||||
def _scan(self):
|
||||
monitor = None
|
||||
try:
|
||||
monitor = r.NativeMonitor(self.policy, b)
|
||||
self.state = (b.boottime(), monitor.report(), None)
|
||||
while not self.stop.wait(self.policy['check_interval_seconds']):
|
||||
monitor.check(force=True)
|
||||
self.state = (b.boottime(), monitor.report(), None)
|
||||
except BaseException as error:
|
||||
completed, report, _ = self.state
|
||||
report = getattr(error, 'report', monitor.report() if monitor is not None else report)
|
||||
self.state = (completed, report, repr(error))
|
||||
|
||||
def check(self):
|
||||
completed, report, error = self.state
|
||||
if error:
|
||||
raise RuntimeError('preparation_native_monitor: ' + error)
|
||||
if completed is not None and b.boottime() - completed > 2:
|
||||
raise RuntimeError('preparation_native_scan_stale_over_2_seconds')
|
||||
return report
|
||||
|
||||
|
||||
def write(root, name, value, *, final=False):
|
||||
payload = b.encode(value)
|
||||
reserve = 0 if final else b.HEADROOM_BYTES + b.STDOUT_ALLOWANCE
|
||||
if b.artifact_size(root, exclude_native=True) + len(payload) > b.CAP_BYTES - reserve:
|
||||
raise RuntimeError('preparation_aggregate_output_cap')
|
||||
b.durable_write(root / name, payload)
|
||||
|
||||
|
||||
def without_core_dumps(command):
|
||||
"""Child-only soft/hard RLIMIT_CORE=0; neither a crash fix nor global policy.
|
||||
|
||||
Use an exec shell rather than unsafe preexec_fn in this threaded supervisor.
|
||||
Piped OS crash handlers may ignore RLIMIT_CORE; old evidence is untouched.
|
||||
"""
|
||||
return ['/usr/bin/bash', '-c',
|
||||
'ulimit -S -c 0 && ulimit -H -c 0 || exit 125; exec "$@"',
|
||||
'preparation-no-core', *command]
|
||||
|
||||
|
||||
def postexit_identity(root, frozen_before, protocol=r.PROTOCOLS[0]):
|
||||
"""External owner: require durable child evidence, independently rehash files."""
|
||||
output = root / preparation_name(protocol)
|
||||
manifest = (output / 'binding.json').read_bytes()
|
||||
report = json.loads((output / 'preparation.json').read_bytes())
|
||||
if (report.get('preclose_identity_verified') is not True
|
||||
or report.get('manifest_sha256_for_independent_review') != r.hashlib.sha256(manifest).hexdigest()
|
||||
or (output / 'preparation-failure.json').exists()):
|
||||
raise ValueError('preparation_missing_or_failed_preclose_identity')
|
||||
evidence = json.loads((output / 'preclose-filesystem.json').read_bytes())
|
||||
independent = r.bound_filesystem_inputs(manifest)
|
||||
independent['files'].update(frozen_before)
|
||||
if evidence != independent:
|
||||
raise ValueError('preparation_postexit_bound_inputs_changed')
|
||||
return r.verify_filesystem_inputs(evidence)
|
||||
|
||||
|
||||
def collect_postexit_identity(root, frozen_before, child, errors, deadline, protocol=r.PROTOCOLS[0]):
|
||||
if child is None:
|
||||
return None
|
||||
result = []
|
||||
|
||||
def rehash():
|
||||
try:
|
||||
result.append((postexit_identity(root, frozen_before, protocol), None))
|
||||
except BaseException as error:
|
||||
result.append((None, repr(error)))
|
||||
|
||||
# File I/O must not take deadline ownership away from the sole supervisor.
|
||||
thread = threading.Thread(target=rehash, daemon=True, name='postexit-identity')
|
||||
thread.start()
|
||||
while thread.is_alive():
|
||||
remaining = deadline - b.boottime()
|
||||
if remaining <= 0:
|
||||
errors.append('preparation_postexit_identity_deadline')
|
||||
return None
|
||||
thread.join(min(.05, remaining))
|
||||
report, error = result[0]
|
||||
if error:
|
||||
errors.append(error)
|
||||
return report
|
||||
|
||||
|
||||
def preparation_name(protocol):
|
||||
return 'prep-' + r.protocol_stage(protocol).lower()
|
||||
|
||||
|
||||
def supervise(root, policy_path, trusted_policy_digest, protocol=r.PROTOCOLS[0]):
|
||||
name = preparation_name(protocol)
|
||||
if r.resource_profile(protocol) is not None:
|
||||
b.extended_historical_calls(root) # same finite root allowlist as reservation
|
||||
started = b.boottime()
|
||||
deadline = started + TOTAL_SECONDS
|
||||
if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'):
|
||||
raise RuntimeError('preparation_pidfd_required')
|
||||
r.protect_artifact_path(root)
|
||||
if any((root / name).exists() for name in ('prep-p1', 'prep-p2', 'native/prep-p1', 'native/prep-p2', 'preparation-start.json', 'budget.json')):
|
||||
raise RuntimeError('preparation_requires_fresh_campaign_no_retry')
|
||||
policy = r.storage_policy_check(SimpleNamespace(
|
||||
artifact_root=root, protocol=protocol, storage_policy=policy_path,
|
||||
trusted_storage_policy_sha256=trusted_policy_digest), b)
|
||||
if policy['not_after_boottime'] < deadline:
|
||||
raise RuntimeError('preparation_policy_expiry_before_cleanup')
|
||||
owned = r.OwnedProcesses()
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise RuntimeError('preparation_subreaper_setup_failed')
|
||||
before = b.digest(CANDIDATE)
|
||||
frozen_before = {str(p): b.digest(p) for p in CANDIDATE.parent.rglob('*') if p.is_file()}
|
||||
write(root, 'preparation-start.json', {
|
||||
'started_boottime': started, 'deadline_boottime': deadline,
|
||||
'prepare_cutoff_boottime': started + PREPARE_SECONDS,
|
||||
'supervisor_pid_starttime': owned.supervisor, 'dedicated_sole_spawner': True,
|
||||
'candidate_before_sha256': before, 'kit_only_attempt': True,
|
||||
'actual_run_reservations_consumed': 0, 'execution_authorized': False,
|
||||
'storage_policy_sha256': trusted_policy_digest, 'policy_role': 'preparation-monitor-only',
|
||||
'supervisor_sha256': b.digest(Path(__file__)), 'runner_sha256': b.digest(ROOT / r.SCRIPT),
|
||||
'independent_native_scan_deadline_owner': True,
|
||||
'stdout_limit_bytes': b.STDOUT_BYTES, 'cleanup_reserve_bytes': b.HEADROOM_BYTES,
|
||||
'stdout_allowance_bytes': b.STDOUT_ALLOWANCE,
|
||||
'child_rlimit_core': {'soft': 0, 'hard': 0, 'scope': 'launched child and descendants only'},
|
||||
})
|
||||
monitor = AsyncNativeMonitor(policy)
|
||||
child = capture = None
|
||||
command = None
|
||||
clean = False
|
||||
errors = []
|
||||
interrupted = threading.Event()
|
||||
handlers = {sig: signal.signal(sig, lambda *_: interrupted.set()) for sig in (signal.SIGTERM, signal.SIGINT)}
|
||||
native_report = None
|
||||
launch_started = launch_ended = None
|
||||
|
||||
def check():
|
||||
nonlocal native_report
|
||||
if interrupted.is_set():
|
||||
raise RuntimeError('preparation_interrupted')
|
||||
if b.boottime() >= started + PREPARE_SECONDS:
|
||||
raise RuntimeError('preparation_480s_deadline')
|
||||
native_report = monitor.check()
|
||||
if b.artifact_size(root, exclude_native=True) > b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE:
|
||||
raise RuntimeError('preparation_aggregate_output_cap')
|
||||
|
||||
def cleanup_pump():
|
||||
nonlocal native_report
|
||||
if capture is not None:
|
||||
try:
|
||||
capture.pump()
|
||||
except Exception as error:
|
||||
text = repr(error)
|
||||
if text not in errors:
|
||||
errors.append(text)
|
||||
try:
|
||||
native_report = monitor.check()
|
||||
except Exception as error:
|
||||
text = repr(error)
|
||||
if text not in errors:
|
||||
errors.append(text)
|
||||
|
||||
try:
|
||||
while native_report is None:
|
||||
check()
|
||||
time.sleep(.05)
|
||||
write(root, 'native-baseline.json', native_report)
|
||||
native = root / 'native'
|
||||
if not native.exists():
|
||||
b.durable_directory(native)
|
||||
support = native / 'preparation-support'
|
||||
b.durable_directory(support)
|
||||
redirects = r.native_redirects(support, preparation=True)
|
||||
for directory in redirects.values():
|
||||
b.durable_directory(Path(directory))
|
||||
environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
|
||||
**redirects)
|
||||
for key in ('CONDA_PREFIX', 'VIRTUAL_ENV', 'PYTHONEXE', 'PYTHONHOME', 'args'):
|
||||
environment.pop(key, None)
|
||||
environment['PYTHONPATH'] = ':'.join([
|
||||
str(ROOT / 'source/SkillFactory'),
|
||||
*['/home/cen/IsaacLab/source/' + name for name in
|
||||
('isaaclab', 'isaaclab_physx', 'isaaclab_assets', 'isaaclab_tasks')]])
|
||||
arguments = [str(ROOT / r.SCRIPT), '--prepare-binding', '--discover-inventory',
|
||||
'--protocol', protocol, '--trial-index', '0', '--device', 'cuda:0',
|
||||
'--artifact-root', str(root), '--output', str(root / name)]
|
||||
command = without_core_dumps(r.launch_command(arguments, environment))
|
||||
write(root, 'preparation-command.json', {'argv': command, 'cwd': str(ROOT),
|
||||
'environment_overrides': {k: environment[k] for k in ('HOME', 'TMPDIR', 'XDG_CACHE_HOME',
|
||||
'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'PYTHONPATH', 'PYTHONDONTWRITEBYTECODE',
|
||||
'PYTEST_DISABLE_PLUGIN_AUTOLOAD')}, 'launch_identity': r.launch_identity()})
|
||||
check()
|
||||
launch_started = b.boottime()
|
||||
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, start_new_session=True)
|
||||
try:
|
||||
owned.attach(child)
|
||||
except BaseException:
|
||||
owned.unknown = True
|
||||
raise
|
||||
capture = r.StdoutCapture(child.stdout, root / 'preparation.stdout.log', b)
|
||||
while not capture.eof or child.poll() is None:
|
||||
capture.check()
|
||||
check()
|
||||
owned.discover()
|
||||
time.sleep(.02)
|
||||
launch_ended = b.boottime()
|
||||
if child.returncode != 0:
|
||||
raise RuntimeError('preparation_child_exit_' + str(child.returncode))
|
||||
except BaseException as error:
|
||||
errors.append(repr(error))
|
||||
finally:
|
||||
try:
|
||||
if child is not None:
|
||||
try:
|
||||
if capture is None and child.stdout is not None:
|
||||
child.stdout.close()
|
||||
finally:
|
||||
clean = r.cleanup_group(child, deadline, b, owned, cleanup_pump)
|
||||
else:
|
||||
clean = not owned.discover() and not owned.unknown
|
||||
if capture is not None:
|
||||
capture.close()
|
||||
if capture.discarded or capture.error:
|
||||
errors.append('preparation_stdout_incomplete')
|
||||
native_report = monitor.check()
|
||||
except BaseException as error:
|
||||
errors.append(repr(error))
|
||||
finally:
|
||||
monitor.stop.set() # never join a blocked filesystem scanner
|
||||
for sig, handler in handlers.items():
|
||||
signal.signal(sig, handler)
|
||||
postexit = collect_postexit_identity(root, frozen_before, child, errors, deadline, protocol)
|
||||
after = b.digest(CANDIDATE)
|
||||
if after != before:
|
||||
errors.append('candidate_changed')
|
||||
remaining = owned.discover()
|
||||
report = {
|
||||
'execution_authorized': False, 'kit_only_launch_started': child is not None,
|
||||
'actual_run_reservations_consumed': 0, 'command_record': 'preparation-command.json' if command else None,
|
||||
'started_boottime': started, 'finished_boottime': b.boottime(), 'deadline_boottime': deadline,
|
||||
'launch_started_boottime': launch_started, 'launch_ended_boottime': launch_ended,
|
||||
'returncode': child.returncode if child is not None else None,
|
||||
'postexit_identity': postexit,
|
||||
'identity_ownership': 'child actual cfg preclose; parent immutable files and process cleanup postexit',
|
||||
'cleanup_clean': clean, 'ownership_unknown': owned.unknown,
|
||||
'owned_remaining': remaining, 'owned_identities': sorted(owned.known),
|
||||
'stdout_bytes': capture.written if capture else 0, 'stdout_discarded': capture.discarded if capture else 0,
|
||||
'stdout_eof': capture.eof if capture else False, 'errors': errors,
|
||||
'monitor': monitor.state[1], 'monitor_last_completed_boottime': monitor.state[0],
|
||||
'monitor_thread_still_alive': monitor.thread.is_alive(),
|
||||
'candidate_before_sha256': before, 'candidate_after_sha256': after,
|
||||
'protocol': protocol, 'preparation_directory': name,
|
||||
'outcomes': {filename: (root / name / filename).is_file()
|
||||
for filename in ('inventory.proposed.json', 'binding.json', 'preparation.json')},
|
||||
'artifact_bytes_before_report': b.artifact_size(root, exclude_native=True),
|
||||
'scope': b.STORAGE_SCOPE, 'native_hard_cap': False,
|
||||
'inventory_and_manifest_trust': 'unreviewed proposals only; no execution identity approved',
|
||||
}
|
||||
write(root, 'preparation-supervisor.json', report, final=True)
|
||||
return 0 if child is not None and clean and not remaining and not errors else 1
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--protocol', choices=r.PROTOCOLS, default=r.PROTOCOLS[0])
|
||||
parser.add_argument('--artifact-root', type=Path, required=True)
|
||||
parser.add_argument('--storage-policy', type=Path, required=True)
|
||||
parser.add_argument('--trusted-storage-policy-sha256', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
return supervise(args.artifact_root, args.storage_policy, args.trusted_storage_policy_sha256, args.protocol)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user