Files
SkillFactory/scripts/prepare_cycle_sim_binding.py
T
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

311 lines
14 KiB
Python

#!/usr/bin/env python3
"""Dedicated ONE preparation-only supervisor; never an execution grant or retry.
Native scans run on a daemon thread, never on the BOOTTIME deadline owner.
The main thread alone spawns, discovers, signals and reaps the Kit process tree.
"""
from __future__ import annotations
import argparse
import ctypes
import importlib.util
import json
import os
import signal
import subprocess
import threading
import time
from pathlib import Path
from types import SimpleNamespace
ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location('preparation_runner', ROOT / 'scripts/evaluate_cycle_sim_residual.py')
r = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(r)
b = r.cpu_module('evaluation_budget')
TOTAL_SECONDS = 600
PREPARE_SECONDS = 480
CANDIDATE = ROOT / 'logs/contact_cycle/sim_residual_training_v1/prepared01/candidate_u0256.pt'
class AsyncNativeMonitor:
"""A blocked native scan cannot block the sole deadline/cleanup owner."""
def __init__(self, policy):
self.state = (None, None, None)
self.stop = threading.Event()
self.policy = policy
self.thread = threading.Thread(target=self._scan, daemon=True, name='native-monitor')
self.thread.start()
def _scan(self):
monitor = None
try:
monitor = r.NativeMonitor(self.policy, b)
self.state = (b.boottime(), monitor.report(), None)
while not self.stop.wait(self.policy['check_interval_seconds']):
monitor.check(force=True)
self.state = (b.boottime(), monitor.report(), None)
except BaseException as error:
completed, report, _ = self.state
report = getattr(error, 'report', monitor.report() if monitor is not None else report)
self.state = (completed, report, repr(error))
def check(self):
completed, report, error = self.state
if error:
raise RuntimeError('preparation_native_monitor: ' + error)
if completed is not None and b.boottime() - completed > 2:
raise RuntimeError('preparation_native_scan_stale_over_2_seconds')
return report
def write(root, name, value, *, final=False):
payload = b.encode(value)
reserve = 0 if final else b.HEADROOM_BYTES + b.STDOUT_ALLOWANCE
if b.artifact_size(root, exclude_native=True) + len(payload) > b.CAP_BYTES - reserve:
raise RuntimeError('preparation_aggregate_output_cap')
b.durable_write(root / name, payload)
def without_core_dumps(command):
"""Child-only soft/hard RLIMIT_CORE=0; neither a crash fix nor global policy.
Use an exec shell rather than unsafe preexec_fn in this threaded supervisor.
Piped OS crash handlers may ignore RLIMIT_CORE; old evidence is untouched.
"""
return ['/usr/bin/bash', '-c',
'ulimit -S -c 0 && ulimit -H -c 0 || exit 125; exec "$@"',
'preparation-no-core', *command]
def postexit_identity(root, frozen_before, protocol=r.PROTOCOLS[0]):
"""External owner: require durable child evidence, independently rehash files."""
output = root / preparation_name(protocol)
manifest = (output / 'binding.json').read_bytes()
report = json.loads((output / 'preparation.json').read_bytes())
if (report.get('preclose_identity_verified') is not True
or report.get('manifest_sha256_for_independent_review') != r.hashlib.sha256(manifest).hexdigest()
or (output / 'preparation-failure.json').exists()):
raise ValueError('preparation_missing_or_failed_preclose_identity')
evidence = json.loads((output / 'preclose-filesystem.json').read_bytes())
independent = r.bound_filesystem_inputs(manifest)
independent['files'].update(frozen_before)
if evidence != independent:
raise ValueError('preparation_postexit_bound_inputs_changed')
return r.verify_filesystem_inputs(evidence)
def collect_postexit_identity(root, frozen_before, child, errors, deadline, protocol=r.PROTOCOLS[0]):
if child is None:
return None
result = []
def rehash():
try:
result.append((postexit_identity(root, frozen_before, protocol), None))
except BaseException as error:
result.append((None, repr(error)))
# File I/O must not take deadline ownership away from the sole supervisor.
thread = threading.Thread(target=rehash, daemon=True, name='postexit-identity')
thread.start()
while thread.is_alive():
remaining = deadline - b.boottime()
if remaining <= 0:
errors.append('preparation_postexit_identity_deadline')
return None
thread.join(min(.05, remaining))
report, error = result[0]
if error:
errors.append(error)
return report
def preparation_name(protocol):
return 'prep-' + r.protocol_stage(protocol).lower()
def supervise(root, policy_path, trusted_policy_digest, protocol=r.PROTOCOLS[0]):
name = preparation_name(protocol)
if r.resource_profile(protocol) is not None:
b.extended_historical_calls(root) # same finite root allowlist as reservation
started = b.boottime()
deadline = started + TOTAL_SECONDS
if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'):
raise RuntimeError('preparation_pidfd_required')
r.protect_artifact_path(root)
if any((root / name).exists() for name in ('prep-p1', 'prep-p2', 'native/prep-p1', 'native/prep-p2', 'preparation-start.json', 'budget.json')):
raise RuntimeError('preparation_requires_fresh_campaign_no_retry')
policy = r.storage_policy_check(SimpleNamespace(
artifact_root=root, protocol=protocol, storage_policy=policy_path,
trusted_storage_policy_sha256=trusted_policy_digest), b)
if policy['not_after_boottime'] < deadline:
raise RuntimeError('preparation_policy_expiry_before_cleanup')
owned = r.OwnedProcesses()
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
raise RuntimeError('preparation_subreaper_setup_failed')
before = b.digest(CANDIDATE)
frozen_before = {str(p): b.digest(p) for p in CANDIDATE.parent.rglob('*') if p.is_file()}
write(root, 'preparation-start.json', {
'started_boottime': started, 'deadline_boottime': deadline,
'prepare_cutoff_boottime': started + PREPARE_SECONDS,
'supervisor_pid_starttime': owned.supervisor, 'dedicated_sole_spawner': True,
'candidate_before_sha256': before, 'kit_only_attempt': True,
'actual_run_reservations_consumed': 0, 'execution_authorized': False,
'storage_policy_sha256': trusted_policy_digest, 'policy_role': 'preparation-monitor-only',
'supervisor_sha256': b.digest(Path(__file__)), 'runner_sha256': b.digest(ROOT / r.SCRIPT),
'independent_native_scan_deadline_owner': True,
'stdout_limit_bytes': b.STDOUT_BYTES, 'cleanup_reserve_bytes': b.HEADROOM_BYTES,
'stdout_allowance_bytes': b.STDOUT_ALLOWANCE,
'child_rlimit_core': {'soft': 0, 'hard': 0, 'scope': 'launched child and descendants only'},
})
monitor = AsyncNativeMonitor(policy)
child = capture = None
command = None
clean = False
errors = []
interrupted = threading.Event()
handlers = {sig: signal.signal(sig, lambda *_: interrupted.set()) for sig in (signal.SIGTERM, signal.SIGINT)}
native_report = None
launch_started = launch_ended = None
def check():
nonlocal native_report
if interrupted.is_set():
raise RuntimeError('preparation_interrupted')
if b.boottime() >= started + PREPARE_SECONDS:
raise RuntimeError('preparation_480s_deadline')
native_report = monitor.check()
if b.artifact_size(root, exclude_native=True) > b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE:
raise RuntimeError('preparation_aggregate_output_cap')
def cleanup_pump():
nonlocal native_report
if capture is not None:
try:
capture.pump()
except Exception as error:
text = repr(error)
if text not in errors:
errors.append(text)
try:
native_report = monitor.check()
except Exception as error:
text = repr(error)
if text not in errors:
errors.append(text)
try:
while native_report is None:
check()
time.sleep(.05)
write(root, 'native-baseline.json', native_report)
native = root / 'native'
if not native.exists():
b.durable_directory(native)
support = native / 'preparation-support'
b.durable_directory(support)
redirects = r.native_redirects(support, preparation=True)
for directory in redirects.values():
b.durable_directory(Path(directory))
environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
**redirects)
for key in ('CONDA_PREFIX', 'VIRTUAL_ENV', 'PYTHONEXE', 'PYTHONHOME', 'args'):
environment.pop(key, None)
environment['PYTHONPATH'] = ':'.join([
str(ROOT / 'source/SkillFactory'),
*['/home/cen/IsaacLab/source/' + name for name in
('isaaclab', 'isaaclab_physx', 'isaaclab_assets', 'isaaclab_tasks')]])
arguments = [str(ROOT / r.SCRIPT), '--prepare-binding', '--discover-inventory',
'--protocol', protocol, '--trial-index', '0', '--device', 'cuda:0',
'--artifact-root', str(root), '--output', str(root / name)]
command = without_core_dumps(r.launch_command(arguments, environment))
write(root, 'preparation-command.json', {'argv': command, 'cwd': str(ROOT),
'environment_overrides': {k: environment[k] for k in ('HOME', 'TMPDIR', 'XDG_CACHE_HOME',
'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'PYTHONPATH', 'PYTHONDONTWRITEBYTECODE',
'PYTEST_DISABLE_PLUGIN_AUTOLOAD')}, 'launch_identity': r.launch_identity()})
check()
launch_started = b.boottime()
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, start_new_session=True)
try:
owned.attach(child)
except BaseException:
owned.unknown = True
raise
capture = r.StdoutCapture(child.stdout, root / 'preparation.stdout.log', b)
while not capture.eof or child.poll() is None:
capture.check()
check()
owned.discover()
time.sleep(.02)
launch_ended = b.boottime()
if child.returncode != 0:
raise RuntimeError('preparation_child_exit_' + str(child.returncode))
except BaseException as error:
errors.append(repr(error))
finally:
try:
if child is not None:
try:
if capture is None and child.stdout is not None:
child.stdout.close()
finally:
clean = r.cleanup_group(child, deadline, b, owned, cleanup_pump)
else:
clean = not owned.discover() and not owned.unknown
if capture is not None:
capture.close()
if capture.discarded or capture.error:
errors.append('preparation_stdout_incomplete')
native_report = monitor.check()
except BaseException as error:
errors.append(repr(error))
finally:
monitor.stop.set() # never join a blocked filesystem scanner
for sig, handler in handlers.items():
signal.signal(sig, handler)
postexit = collect_postexit_identity(root, frozen_before, child, errors, deadline, protocol)
after = b.digest(CANDIDATE)
if after != before:
errors.append('candidate_changed')
remaining = owned.discover()
report = {
'execution_authorized': False, 'kit_only_launch_started': child is not None,
'actual_run_reservations_consumed': 0, 'command_record': 'preparation-command.json' if command else None,
'started_boottime': started, 'finished_boottime': b.boottime(), 'deadline_boottime': deadline,
'launch_started_boottime': launch_started, 'launch_ended_boottime': launch_ended,
'returncode': child.returncode if child is not None else None,
'postexit_identity': postexit,
'identity_ownership': 'child actual cfg preclose; parent immutable files and process cleanup postexit',
'cleanup_clean': clean, 'ownership_unknown': owned.unknown,
'owned_remaining': remaining, 'owned_identities': sorted(owned.known),
'stdout_bytes': capture.written if capture else 0, 'stdout_discarded': capture.discarded if capture else 0,
'stdout_eof': capture.eof if capture else False, 'errors': errors,
'monitor': monitor.state[1], 'monitor_last_completed_boottime': monitor.state[0],
'monitor_thread_still_alive': monitor.thread.is_alive(),
'candidate_before_sha256': before, 'candidate_after_sha256': after,
'protocol': protocol, 'preparation_directory': name,
'outcomes': {filename: (root / name / filename).is_file()
for filename in ('inventory.proposed.json', 'binding.json', 'preparation.json')},
'artifact_bytes_before_report': b.artifact_size(root, exclude_native=True),
'scope': b.STORAGE_SCOPE, 'native_hard_cap': False,
'inventory_and_manifest_trust': 'unreviewed proposals only; no execution identity approved',
}
write(root, 'preparation-supervisor.json', report, final=True)
return 0 if child is not None and clean and not remaining and not errors else 1
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--protocol', choices=r.PROTOCOLS, default=r.PROTOCOLS[0])
parser.add_argument('--artifact-root', type=Path, required=True)
parser.add_argument('--storage-policy', type=Path, required=True)
parser.add_argument('--trusted-storage-policy-sha256', required=True)
args = parser.parse_args(argv)
return supervise(args.artifact_root, args.storage_policy, args.trusted_storage_policy_sha256, args.protocol)
if __name__ == '__main__':
raise SystemExit(main())