#!/usr/bin/env python3 """Prospective P1/P2 evaluator. CPU --check-only; Kit --prepare-binding; gated --run. No mode grants its own permission. --run is a Linux supervisor, not a direct Kit invocation. It consumes a fresh independently trusted parent grant and binding. Never use an old replay grant. See logs/contact_cycle/evaluation_runner_v1/README.md and docs/contact_cycle_native_monitor_v3.md (superseding native coverage policy). """ from __future__ import annotations import argparse import ctypes import dataclasses import enum import errno import functools import hashlib import importlib import importlib.util import inspect import json import math import os import shlex import signal import socket import stat import struct import subprocess import sys import time from contextlib import ExitStack, suppress from dataclasses import asdict from pathlib import Path, PosixPath, WindowsPath from types import GetSetDescriptorType, MappingProxyType, MemberDescriptorType, ModuleType, SimpleNamespace ROOT = Path(__file__).resolve().parents[1] SCRIPT = 'scripts/evaluate_cycle_sim_residual.py' PACKAGE = 'source/SkillFactory/SkillFactory/' TASK = 'SkillFactory.tasks.manager_based.contact_cycle_sim.' LEG = 'SkillFactory.legs.contact_cycle_sim.' PROTOCOLS = ('configs/contact_cycle/continuous_eval_p1_v1.json', 'configs/contact_cycle/bounded_diagnostic_p2_v1.json', 'configs/contact_cycle/bounded_diagnostic_p2_extended_v1.json') FUTURE_SOURCES = { 'runner': [SCRIPT, PACKAGE + 'legs/contact_cycle_sim/evaluation_budget.py'], 'instrumentation': [PACKAGE + 'tasks/manager_based/contact_cycle_sim/' + name + '.py' for name in ('evaluation_instrumentation', 'evaluation_rng_scope')], } REAL_TEST = 'tests/test_contact_cycle_sim_replay.py::test_actual_frozen_candidate_cpu_only_when_artifacts_present' LAUNCHER = '/home/cen/IsaacLab/isaaclab.sh' OPTIONAL_REPLICATOR = 'omni.replicator.core' # Exact constructor-late imports evidenced in p1_first_failure_diagnosis.md. # Full source identities are checked before import and retained in the binding. CONSTRUCTOR_IMPORT_PINS = { 'isaaclab.cloner.cloner_cfg': ( '/home/cen/IsaacLab/source/isaaclab/isaaclab/cloner/cloner_cfg.py', 'f48e268609e8e5e91e30b8e6d364cc76579aeb497d69f963cd45975152a282ca'), 'isaaclab.cloner.cloner_strategies': ( '/home/cen/IsaacLab/source/isaaclab/isaaclab/cloner/cloner_strategies.py', 'ea5943bb5e17df09b792dcf7de436f7481daed8768dc2d8573c6eb45464481d8'), 'isaaclab.sim.schemas.schemas_actuators': ( '/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/schemas/schemas_actuators.py', '3defa9a6ff41d39e797c6d4a1dfee031913fdaf64ff7f99305a844f4ddf47f77'), 'isaaclab.sim.spawners.wrappers': ( '/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/spawners/wrappers/__init__.py', '431dfebf1f14464b5479fe618720d913fdefb8e1d613a0c54f45dcddb671cce5'), 'isaaclab.sim.spawners.wrappers.wrappers_cfg': ( '/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/spawners/wrappers/wrappers_cfg.py', 'ce073f03584f9cab400735fce1af92b261b0e9217592c7ce79c1b8f541399574'), 'isaaclab.sim.views.usd_frame_view': ( '/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/views/usd_frame_view.py', '6c186fe2991a608323f12f40fb67516dd629079eabe4f6e525c198e0452a59d9'), 'isaaclab.utils.warp.fabric': ( '/home/cen/IsaacLab/source/isaaclab/isaaclab/utils/warp/fabric.py', '3756853121fbd0fb88b01f07c521927e9cdc1ccd96ceb40b6a603bef29e56c94'), 'isaaclab_physx.assets.surface_gripper': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/assets/surface_gripper/__init__.py', '42992defc933ea096bac4666c574bfd32ace19fb3886f6e41b5d5498e06da2ef'), 'isaaclab_physx.assets.surface_gripper.surface_gripper_cfg': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/assets/surface_gripper/surface_gripper_cfg.py', 'c71846dad68db328c6faa9ac7dfe37488502e9477f56de0961ab3648f273b14a'), 'isaaclab_physx.cloner': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/__init__.py', '13e3966a049c8a6feeee9a8018162ddce6667f9cbdc0d7350644b2176a191def'), 'isaaclab_physx.cloner.physx_replicate': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/physx_replicate.py', 'a0e7eb7f47e45ce27351002a0ef6c137ddb3c1c4a20a5efcbbd389c3ff5a426c'), 'isaaclab_physx.sensors': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/__init__.py', '52dc0bed9f1cc96b938b9a6aae4f2f48076006fce0663c2e8ad017488f8098a4'), 'isaaclab_physx.sensors.contact_sensor': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/__init__.py', '2f81a5de0c1a2a264fc18832fbfe33e3cb49a41a7a246b0de4471437dcbf3dfd'), 'isaaclab_physx.sensors.contact_sensor.contact_sensor': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py', '09507db17049bcadd6a48f5992bc20f79b019356c13f1ba3ef1527b2e4188b2c'), 'isaaclab_physx.sensors.contact_sensor.contact_sensor_data': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor_data.py', '711c461be5c696a7a99f0a8fd2a010cd9a5a47e1b8bcfdf493e5ac8af8b65e3d'), 'isaaclab_physx.sensors.contact_sensor.kernels': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/kernels.py', 'e3e9d678336a704a9ba58fe7d31fe8a3a577f66f357c921a2c312c2edb8fa6e3'), 'isaaclab_physx.sensors.joint_wrench': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/__init__.py', '308cb956e92e065c05b66ff0ec0c139dc3bc35380a09d58191af13acf5c14f36'), 'isaaclab_physx.sensors.joint_wrench.joint_wrench_sensor': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/joint_wrench_sensor.py', 'eff63d6f6e8c9958651280889c1797c9cbf2a14fb1b35a1f4e39cb208399ee5f'), 'isaaclab_physx.sensors.joint_wrench.joint_wrench_sensor_data': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/joint_wrench_sensor_data.py', '32b168096dc384587e2962b8e09caccba1d76bb8e40459073b262843b6511937'), 'isaaclab_physx.sensors.joint_wrench.kernels': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/kernels.py', 'a71e33ae91ab7bef9d585d7ca8137028a3dfe5b169dabec4b9b763f8497d25a0'), 'isaaclab_physx.sensors.kernels': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/kernels.py', '297f79677d58d7c5ed38cffc7ba19480bc3236a331a64700a54d57f34a5b8b56'), 'isaaclab_physx.sim.views': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sim/views/__init__.py', 'ac57ce5aafa903fe42868a410461b9db8fc8a150ed458f9b8bfe80772a94f97c'), 'isaaclab_physx.sim.views.fabric_frame_view': ( '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sim/views/fabric_frame_view.py', '7d9d199be7dc8b9087c60f9f89567fb0463fd0269d20b0aa0e20ae7f8ac1d329'), } # One reviewed installed shell/CLI layout only; the bridge changes subprocess.run # to exec, not the inherited socket authentication or the runtime import tree. LAUNCH_PINS = { '/home/cen/IsaacLab/isaaclab.sh': 'c3a6f3111012aa39c5f568012da1ef2cde52a80cbc2277bc59ce295895d7bc68', '/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/__init__.py': '8886104cee16eb41c8c7dccbe2955cf8e4c72677f49423bf586f4eb4615a96af', '/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/envs.py': '92002ebc7301f559cd17dcbf13a749006c1353aabf9f1247e57a9438a85acecd', '/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/format.py': 'bc575ef81dff01a4c8a7b11496515ac34fa9bbdf14613978e84fe37449cd64cc', '/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/install.py': 'b285da1f19beada43c1a4ad9aa7e3b432a0042eb0074c7c89ccc7aa0d3c46b31', '/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/misc.py': 'b608ffecbae98c810024dc361852325d09197a9f9a355b270bd60c410a459747', '/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/utils.py': '9608423230d159234e740a8000fd8206a4c980311e6517f97d785eb58f68fc97', '/home/cen/IsaacLab/source/isaaclab/isaaclab/__init__.py': '24e50c4e57b9be1c55aa949f1b8dd416d95e6002e12941abe33a7a6eb16f697b', '/home/cen/isaacsim-6.0.1/python.sh': '37f7e3717f1fce1800d0b15b55fcd754c909978bd2455b9d6de3d22d77a92bc3', '/home/cen/isaacsim-6.0.1/setup_python_env.sh': '0f73e6dac65efa1a28f994939f4734859d2c39eb236c658bfad9fab70c8882e6', '/home/cen/isaacsim-6.0.1/kit/python/bin/python3': 'f3300e753c44afd7860b3f26ccf7c3d98c323b4fc90e7ce2fccae2cd49efd09c', '/home/cen/isaacsim-6.0.1/kit/libcarb.so': 'aef0445bb75bd3da04acfabf6d067fb9e40b5c3d17f07dd5369bd63c75489c4f', '/usr/bin/python3.12': 'e50d468e8b0adfb05733f5b87b3cff34829c4a8c1aea50c865aa8bdfe4bb150f', '/usr/bin/bash': 'bc5945feb8bd26203ebfafea5ce1878bb2e32cb8fb50ab7ae395cfb1e1aaaef1', } RESOLVED_PYTHON = '/home/cen/isaacsim-6.0.1/kit/python/bin/python3' EXEC_BRIDGE = """import os, sys from pathlib import Path from isaaclab.cli.utils import extract_python_exe resolved = extract_python_exe() if Path(resolved).resolve() != Path('/home/cen/isaacsim-6.0.1/python.sh'): raise RuntimeError('identity_unreviewed_resolved_interpreter') os.execv(resolved, [resolved, *sys.argv[1:]]) """ def launch_identity(): b = cpu_module('evaluation_budget') for path, expected in LAUNCH_PINS.items(): if b.digest(path) != expected: raise RuntimeError('identity_launch_source_changed: ' + path) # This layout has no optional setup_conda_env.sh or docker startup helper. sim = Path('/home/cen/IsaacLab/_isaac_sim') if (sim.resolve() != Path('/home/cen/isaacsim-6.0.1') or (sim / 'setup_conda_env.sh').exists() or (sim / 'vulkan_check.sh').exists() or any(Path('/home/cen/IsaacLab', name).exists() for name in ('env_isaaclab', '.venv'))): raise RuntimeError('identity_unreviewed_launch_layout') return {'format': 'skillfactory.fd-exec-launch.v1', 'sources': LAUNCH_PINS, 'bridge_sha256': hashlib.sha256(EXEC_BRIDGE.encode()).hexdigest(), 'runner_sha256': b.digest(ROOT / SCRIPT), 'interpreter': RESOLVED_PYTHON} def launch_command(arguments, environment): launch_identity() if any(environment.get(k) for k in ('VIRTUAL_ENV', 'CONDA_PREFIX', 'PYTHONEXE', 'PYTHONHOME', 'args')): raise RuntimeError('identity_launch_requires_reviewed_no_virtualenv_layout') text = Path(LAUNCHER).read_text() text = text.replace('export ISAACLAB_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"', 'export ISAACLAB_PATH=/home/cen/IsaacLab') old = 'exec "$python_exe" -c "from isaaclab.cli import cli; cli()" "$@"' if text.count(old) != 1: raise RuntimeError('identity_launch_exec_boundary_changed') text = text.replace(old, 'exec "$python_exe" -c ' + shlex.quote(EXEC_BRIDGE) + ' "$@"') return ['/usr/bin/bash', '-c', text, LAUNCHER, *arguments] def cpu_module(name): """Load pure stdlib source without importing Isaac registration before Kit. Budget has its real file-backed module identity (no fake parent packages). CPU-only protocol planning uses a private identity, never a binding snapshot. """ if name == 'evaluation_adoption': cpu_module('evaluation_budget') full = LEG + name if name in ('evaluation_budget', 'evaluation_adoption') else '_evaluation_cpu_' + name if full not in sys.modules: path = ROOT / PACKAGE / 'legs/contact_cycle_sim' / (name + '.py') spec = importlib.util.spec_from_file_location(full, path) module = importlib.util.module_from_spec(spec) sys.modules[full] = module spec.loader.exec_module(module) return sys.modules[full] def protect_artifact_path(path): path = Path(path) allowed = ROOT / 'logs/contact_cycle/evaluation_runs_v1' if (not path.is_absolute() or path != path.resolve() or (path.is_relative_to(ROOT) and not path.is_relative_to(allowed))): raise ValueError('Use new evaluation_runs_v1 campaign paths or canonical paths outside repository; ' 'historic evidence/source/release trees are protected') def read_trusted(path, expected): payload = Path(path).read_bytes() if (len(expected or '') != 64 or any(c not in '0123456789abcdef' for c in expected) or hashlib.sha256(payload).hexdigest() != expected): raise ValueError('identity_independently_trusted_digest_required: ' + str(path)) return payload def resource_profile(protocol_path): if protocol_path not in PROTOCOLS: raise ValueError('Unknown fixed protocol path') protocol = cpu_module('evaluation_protocol').load_protocol(ROOT / protocol_path) return protocol.document.get('resource_profile') def protocol_stage(protocol_path): """Fixed-path stage lookup, safe before real runtime binding. Never load a private CPU planning module into the Kit worker. Full protocol validation remains with the supervisor and the real runtime module. """ if protocol_path not in PROTOCOLS: raise ValueError('Unknown fixed protocol path') return 'P1' if protocol_path == PROTOCOLS[0] else 'P2' def grant_check(grant, args, b): extended = resource_profile(args.protocol) == b.EXTENDED_PROFILE fields = {'format', 'campaign_id', 'launch_index', 'stage', 'trial_indices', 'manifest_sha256', 'protocol_sha256', 'inventory_sha256', 'artifact_root', 'boot_id', 'not_after_boottime', 'parent_review', 'p1_evidence_sha256', 'storage_policy_sha256', 'storage_scope', 'launch_identity_sha256'} if extended: fields = (fields - {'p1_evidence_sha256'}) | {'resource_profile', 'p1_adoption_sha256'} if not isinstance(grant, dict) or set(grant) != fields: raise ValueError('Explicit fresh parent grant schema required') index = grant['launch_index'] if type(index) is not int or index not in ((0,) if extended else (0, 1, 2)): raise ValueError('Parent grant launch slot') expected_stage = 'P2' if extended or index == 2 else 'P1' rows = list(range(20)) if expected_stage == 'P2' else [index] if (grant['format'] != (b.EXTENDED_GRANT_FORMAT if extended else 'skillfactory.evaluation-parent-grant.v2') or not isinstance(grant['campaign_id'], str) or not grant['campaign_id'] or not isinstance(grant['parent_review'], str) or not grant['parent_review'].strip() or grant['stage'] != expected_stage or grant['trial_indices'] != rows or grant['manifest_sha256'] != args.trusted_manifest_sha256 or grant['protocol_sha256'] != b.digest(ROOT / args.protocol) or grant['inventory_sha256'] != b.digest(args.inventory) or grant['storage_policy_sha256'] != args.trusted_storage_policy_sha256 or grant['storage_scope'] != b.STORAGE_SCOPE or grant['launch_identity_sha256'] != hashlib.sha256(b.encode(launch_identity())).hexdigest() or grant['artifact_root'] != str(args.artifact_root) or grant['boot_id'] != b.boot_id() or args.protocol != PROTOCOLS[2 if extended else index == 2] or type(grant['not_after_boottime']) not in (int, float) or not b.boottime() < grant['not_after_boottime'] <= b.boottime() + 3600 or (not extended and index < 2 and grant['p1_evidence_sha256'] != []) or (not extended and index == 2 and (not isinstance(grant['p1_evidence_sha256'], list) or len(grant['p1_evidence_sha256']) != 2))): raise ValueError('Parent grant identity/order/expiry/review mismatch') if extended: if grant['resource_profile'] != b.EXTENDED_PROFILE or args.artifact_root not in (b.EXTENDED_ROOT, b.SECOND_EXTENDED_ROOT): raise ValueError('Parent grant fixed extended profile/root mismatch') adoption = cpu_module('evaluation_adoption').verify_adoption(args.artifact_root, grant['p1_adoption_sha256']) if adoption['source_boot_id'] != b.boot_id(): raise ValueError('Parent grant adoption same-boot mismatch') read_trusted(args.manifest, args.trusted_manifest_sha256) return grant def mount_identity(path): """Read-only mount facts, NOT evidence that a quota or confinement is effective.""" matches = [] for line in Path('/proc/self/mountinfo').read_text().splitlines(): fields = line.split() mount = Path(fields[4].replace('\\040', ' ')) if path == mount or mount in path.parents: matches.append((len(mount.parts), line)) if not matches: raise RuntimeError('storage_mount_identity_unavailable') return max(matches)[1] STORAGE_APPROVAL = 'logs/contact_cycle/evaluation_runner_recovery_v1/USER_STORAGE_SCOPE_AMENDMENT.md' STORAGE_APPROVAL_SHA256 = 'e4fac0d36799c9beb67af7687a390a6b5e8390bde1d4b840a6e26c309578c8db' NATIVE_GAPS = ('sampled logical sizes, not a quota; writes can overshoot between checks', 'unlisted/global cache writers, deleted-open files, symlink targets and native allocations ' 'have unknown bytes/growth; only free-space floors on listed filesystems cover external writes', 'shared native cache changes include unrelated jobs; growth can cause conservative stops', 'stable non-runtime-owned inaccessible shared subtrees are explicit permission gaps; ' 'their bytes and growth are unknown; coverage is incomplete', 'generic HOME/.cache and /tmp are not recursively scanned; unlisted or hidden mounts are uncovered', 'installed data and kit/data/cache/logs are conservative portable-token candidates; ' 'actual token destinations require runtime confirmation; missing candidates are not created') SIM_ROOT = Path('/home/cen/isaacsim-6.0.1') def native_paths(root): # SimulationApp --portable + kit-core.json data/log writers; token resolution # remains unqualified. Never substitute a generic home/cache/tmp tree. return sorted({str(root / 'native'), *shared_native_paths(), *[str(SIM_ROOT / name) for name in ('data', 'kit/data', 'kit/logs')]}) def shared_native_paths(): # Installed clear_caches.sh names these three ov caches explicitly. return sorted({*[str(Path.home() / '.cache/ov' / name) for name in ('shaders', 'texturecache', 'Kit/107.3')], str(SIM_ROOT / 'kit/cache')}) def free_space_paths(root): # Other native roots also observe their own filesystem during their scan. return sorted({str(root), '/tmp', str(Path.home()), str(SIM_ROOT)}) def native_redirects(native, *, preparation=False): redirects = {'HOME': str(native / 'home'), 'XDG_CACHE_HOME': str(native / 'cache'), 'TMPDIR': str(native / 'tmp')} if preparation: redirects.update(XDG_CONFIG_HOME=str(native / 'config'), XDG_DATA_HOME=str(native / 'data')) return redirects def task_owned_cache_paths(root): # Same mapping used to construct the actual subprocess environments. native = root / 'native' return sorted([*native_redirects(native / 'preparation-support', preparation=True).values(), *[path for index in range(3) for path in native_redirects(native / f'launch-{index}').values()]]) def storage_policy_check(args, b): if args.storage_policy is None or args.trusted_storage_policy_sha256 is None: raise RuntimeError('storage_bound_policy_unavailable; no launch permitted') extended = resource_profile(getattr(args, 'protocol', PROTOCOLS[0])) == b.EXTENDED_PROFILE policy = json.loads(read_trusted(args.storage_policy, args.trusted_storage_policy_sha256)) read_trusted(ROOT / STORAGE_APPROVAL, STORAGE_APPROVAL_SHA256) fields = {'format', 'artifact_root', 'limit_bytes', 'scope', 'approval_sha256', 'native_paths', 'check_interval_seconds', 'max_native_growth_bytes', 'min_free_bytes', 'coverage_gaps', 'native_hard_cap', 'boot_id', 'mount_identity', 'not_after_boottime', 'shared_native_paths', 'task_owned_cache_paths', 'free_space_paths', 'free_space_mount_identities'} root = args.artifact_root protect_artifact_path(root) if extended: fields |= {'resource_profile', 'issued_boottime'} if (root not in (b.EXTENDED_ROOT, b.SECOND_EXTENDED_ROOT) or policy.get('resource_profile') != b.EXTENDED_PROFILE or type(policy.get('issued_boottime')) not in (int, float) or not 0 <= policy['issued_boottime'] <= b.boottime() or type(policy.get('not_after_boottime')) not in (int, float) or not policy['issued_boottime'] < policy['not_after_boottime'] <= policy['issued_boottime'] + b.EXTENDED_TOTAL_SECONDS): raise RuntimeError('storage_extended_profile_lifetime_mismatch') if (set(policy) != fields or policy['format'] != ( b.EXTENDED_POLICY_FORMAT if extended else 'skillfactory.output-storage-policy.v4') or not root.is_dir() or policy['artifact_root'] != str(root) or type(policy['limit_bytes']) is not int or not b.HEADROOM_BYTES + b.STDOUT_ALLOWANCE < policy['limit_bytes'] <= b.CAP_BYTES or policy['scope'] != b.STORAGE_SCOPE or policy['approval_sha256'] != STORAGE_APPROVAL_SHA256 or policy['native_paths'] != native_paths(root) or policy['native_hard_cap'] is not False or policy['coverage_gaps'] != list(NATIVE_GAPS) or policy['shared_native_paths'] != shared_native_paths() or policy['task_owned_cache_paths'] != task_owned_cache_paths(root) or policy['free_space_paths'] != free_space_paths(root) or policy['free_space_mount_identities'] != {p: mount_identity(Path(p)) for p in free_space_paths(root)} or any(Path(p).is_relative_to(root) for p in shared_native_paths()) or type(policy['check_interval_seconds']) not in (int, float) or not 0 < policy['check_interval_seconds'] <= 1 or type(policy['max_native_growth_bytes']) is not int or not 0 < policy['max_native_growth_bytes'] <= b.CAP_BYTES or type(policy['min_free_bytes']) is not int or policy['min_free_bytes'] < 2 * b.CAP_BYTES or policy['boot_id'] != b.boot_id() or policy['mount_identity'] != mount_identity(root) or type(policy['not_after_boottime']) not in (int, float) or not b.boottime() < policy['not_after_boottime'] <= b.boottime() + (14400 if extended else 3600)): raise RuntimeError('storage_policy_identity_or_conditions_mismatch') return policy class NativeScanError(RuntimeError): """Retain partial scan facts even when baseline fails before monitor creation.""" def __init__(self, report): self.report = report super().__init__('storage_native_scan_failed: ' + json.dumps(report, sort_keys=True)) def native_metadata(info): return {key: getattr(info, 'st_' + key) for key in ('uid', 'gid', 'mode', 'dev', 'ino', 'ctime_ns')} def native_directory_identity(info): # Creating entries changes ctime during normal writes; it is not a new directory. return (info.st_dev, info.st_ino, info.st_uid, info.st_gid, info.st_mode) def native_directory_names(fd, *, recursive): if not recursive: return [] with os.scandir(fd) as entries: return [entry.name for entry in entries] def native_size(path, *, shared=False, strict_paths=(), exclude_paths=(), required=False, recursive=True): """Descriptor-relative traversal; never follow even a raced ancestor symlink. Only an actual denied directory operation plus stable non-owned metadata and an effective-credential access denial permits a shared coverage gap. A gap's contents are not inspected. recursive=False observes only root/ancestor filesystem facts, never directory entries. Missing ordinary entries are sampled churn, not permission exclusions; disappearance during denial qualification is fatal. """ path = Path(path) strict = tuple(Path(p) for p in strict_paths) excluded = set(map(Path, exclude_paths)) result = {'logical_bytes': 0, 'permission_gaps': [], 'skipped_count': 0, 'disappeared_paths': [], 'symlink_paths': [], 'free_space': None} def fail(current, operation, error, info=None): result['failure'] = {'path': str(current), 'operation': operation, 'errno': error.errno, 'error': str(error), 'metadata': native_metadata(info) if info is not None else None} raise NativeScanError(result) from error def denied(parent, name, current, info, operation, error): # An inaccessible ancestor of a strict path would also hide owned writes. protected = any(current.is_relative_to(p) or p.is_relative_to(current) for p in strict) if (not shared or protected or error.errno not in (errno.EACCES, errno.EPERM) or info.st_uid in (os.getuid(), os.geteuid()) or not stat.S_ISDIR(info.st_mode)): fail(current, operation, error, info) try: after = os.stat(name, dir_fd=parent, follow_symlinks=False) accessible = os.access(name, os.R_OK | os.X_OK, dir_fd=parent, effective_ids=True, follow_symlinks=False) final = os.stat(name, dir_fd=parent, follow_symlinks=False) except OSError as metadata_error: result['denied_operation'] = {'path': str(current), 'errno': error.errno, 'metadata': native_metadata(info)} fail(current, 'permission_metadata_recheck', metadata_error, info) if (native_metadata(info) != native_metadata(after) or native_metadata(info) != native_metadata(final) or accessible): fail(current, 'permission_identity_or_access_changed', error, final) result['permission_gaps'].append({'path': str(current), 'errno': error.errno, 'operation': operation, **native_metadata(final)}) result['skipped_count'] += 1 def observe_free(fd, current): try: usage = os.fstatvfs(fd) result['free_space'] = {'observed_path': str(current), 'free_bytes': usage.f_bavail * usage.f_frsize, 'device': os.fstat(fd).st_dev, 'mount_identity': mount_identity(current)} except OSError as error: fail(current, 'free_space', error) def visit(parent, name, current, *, ancestor=False): if current in excluded: return try: info = os.stat(name, dir_fd=parent, follow_symlinks=False) except FileNotFoundError as error: if required and (ancestor or current == path): fail(current, 'required_directory', error) result['disappeared_paths'].append(str(current)) return except OSError as error: fail(current, 'lstat', error) if stat.S_ISLNK(info.st_mode): # Explicit monitored roots/ancestors and owned trees cannot be aliases. if ancestor or current == path or any(current.is_relative_to(p) or p.is_relative_to(current) for p in strict): fail(current, 'symlink', OSError(errno.ELOOP, 'monitored path is a symlink'), info) result['symlink_paths'].append(str(current)) return if ((ancestor or current == path or any(p.is_relative_to(current) for p in strict)) and not stat.S_ISDIR(info.st_mode)): fail(current, 'directory', OSError(errno.ENOTDIR, 'not a directory'), info) if stat.S_ISREG(info.st_mode): if not os.access(name, os.R_OK, dir_fd=parent, effective_ids=True, follow_symlinks=False): # access() returns False for ENOENT as well as permission denial. # Atomic .pending publication may race the earlier lstat. Only a # descriptor-relative ENOENT recheck is ordinary sampled churn; # retain its last observed bytes conservatively, never a gap. try: os.stat(name, dir_fd=parent, follow_symlinks=False) except FileNotFoundError: result['disappeared_paths'].append(str(current)) result['logical_bytes'] += info.st_size return except OSError as error: fail(current, 'file_access_recheck', error, info) fail(current, 'file_access', PermissionError(None, 'file unreadable; access() exposes no errno'), info) result['logical_bytes'] += info.st_size return if not stat.S_ISDIR(info.st_mode): return fd = None operation = 'open' try: try: fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) operation = 'fstat' if native_directory_identity(os.fstat(fd)) != native_directory_identity(info): fail(current, 'open_identity_changed', OSError(errno.ESTALE, 'directory changed'), info) if ancestor: walk_to_root(fd, current) else: if current == path: observe_free(fd, current) # Scandir errors belong to this directory, not one of its children. operation = 'scandir' names = native_directory_names(fd, recursive=recursive) operation = 'children' for child in names: visit(fd, child, current / child) operation = 'traversal_lstat' after = os.stat(name, dir_fd=parent, follow_symlinks=False) if native_directory_identity(after) != native_directory_identity(info): fail(current, 'traversal_identity_changed', OSError(errno.ESTALE, 'directory changed'), after) except OSError as error: if operation in ('open', 'scandir') and error.errno in (errno.EACCES, errno.EPERM) and not ancestor: denied(parent, name, current, info, operation, error) else: fail(current, operation, error, info) finally: if fd is not None: os.close(fd) def walk_to_root(fd, current): observe_free(fd, current) relative = path.relative_to(current) if not relative.parts: raise ValueError('native scan root must not be /') child = current / relative.parts[0] visit(fd, relative.parts[0], child, ancestor=child != path) try: with ExitStack() as stack: fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) stack.callback(os.close, fd) walk_to_root(fd, Path('/')) except OSError as error: fail(path, 'ancestor', error) return result class NativeMonitor: def __init__(self, policy, b): self.policy, self.b = policy, b self.baseline = self.latest = {} self.baseline_scans = self.latest_scans = {} self.baseline_free_space = self.latest_free_space = {} self.gap_history = [] self.growth = 0 self.next_check = 0 self.latest_scan_completed = False try: self.check(force=True) except Exception as error: if not isinstance(error, NativeScanError): error.report = self.report() raise def check(self, *, force=False): now = self.b.boottime() if not force and now < self.next_check: return self.next_check = now + self.policy['check_interval_seconds'] self.latest_scan_completed = False root = Path(self.policy['artifact_root']) if mount_identity(root) != self.policy['mount_identity']: raise RuntimeError('storage_policy_mount_identity_changed') paths = self.policy['native_paths'] strict = [str(root), *self.policy['task_owned_cache_paths'], *[p for p in paths if p not in self.policy['shared_native_paths']]] scans = {} self.latest_free_space = {} checking_free_space = True try: for path in self.policy['free_space_paths']: if mount_identity(Path(path)) != self.policy['free_space_mount_identities'][path]: raise RuntimeError('storage_policy_free_space_mount_identity_changed: ' + path) observation = native_size(path, required=True, recursive=False) self.latest_free_space[path] = observation['free_space'] if observation['free_space']['mount_identity'] != self.policy['free_space_mount_identities'][path]: raise RuntimeError('storage_policy_free_space_mount_identity_changed: ' + path) if observation['free_space']['free_bytes'] < self.policy['min_free_bytes']: raise RuntimeError('storage_native_free_space_stop') checking_free_space = False # Disjoint scans: output accessibility is NOT native growth/cap accounting. # The unchanged pre-write aggregate cap still excludes only root/native. for path in [str(root), *paths]: exclusions = [p for p in [str(root), *paths] if p != path and Path(p).is_relative_to(path)] scans[path] = native_size(path, shared=path in self.policy['shared_native_paths'], strict_paths=strict, exclude_paths=exclusions, required=path == str(root)) except NativeScanError as error: if checking_free_space: self.latest_free_space[path] = error.report['free_space'] else: scans[path] = error.report self.latest_scans = scans self.latest = {p: scans[p]['logical_bytes'] for p in paths if p in scans} error.report = {**self.report(), 'failed_scan': error.report} raise previous_gaps = {gap['path']: gap for scan in self.latest_scans.values() for gap in scan['permission_gaps']} current_gaps = {gap['path']: gap for scan in scans.values() for gap in scan['permission_gaps']} self.gap_history.append({'boottime': now, 'new': [current_gaps[p] for p in sorted(current_gaps.keys() - previous_gaps.keys())], 'no_longer_observed': [previous_gaps[p] for p in sorted(previous_gaps.keys() - current_gaps.keys())], 'changed': [{'before': previous_gaps[p], 'after': current_gaps[p]} for p in sorted(previous_gaps.keys() & current_gaps.keys()) if previous_gaps[p] != current_gaps[p]]}) self.latest_scan_completed = True self.latest_scans = scans self.latest = {p: scans[p]['logical_bytes'] for p in paths} if not self.baseline_scans: self.baseline_free_space = dict(self.latest_free_space) self.baseline_scans = scans self.baseline = dict(self.latest) self.growth = sum(max(0, size - self.baseline[p]) for p, size in self.latest.items()) if self.gap_history[-1]['changed']: raise RuntimeError('storage_native_permission_gap_identity_changed') if any(scan['free_space']['free_bytes'] < self.policy['min_free_bytes'] for scan in scans.values()): raise RuntimeError('storage_native_free_space_stop') if self.growth >= self.policy['max_native_growth_bytes']: raise RuntimeError('storage_native_growth_stop') def report(self): return {'scope': self.b.STORAGE_SCOPE, 'native_hard_cap': False, 'coverage_complete': False, 'runtime_uid': os.geteuid(), 'latest_scan_completed': self.latest_scan_completed, 'baseline_skipped_count': sum(s['skipped_count'] for s in self.baseline_scans.values()), 'latest_skipped_count': sum(s['skipped_count'] for s in self.latest_scans.values()), 'baseline_logical_bytes': self.baseline, 'latest_logical_bytes': self.latest, 'observed_positive_growth_bytes': self.growth if self.latest_scan_completed else None, 'baseline_scans': self.baseline_scans, 'latest_scans': self.latest_scans, 'baseline_free_space': self.baseline_free_space, 'latest_free_space': self.latest_free_space, 'permission_gap_history': list(self.gap_history), 'policy': self.policy} class ForwardingEvidence: """Stable runtime callbacks, installed before constructor; targets fail closed.""" recorder = None observer = None def __call__(self, event, state): if self.recorder is None: raise RuntimeError('recording_forwarder_uninstalled') return self.recorder(event, state) def seam(self, event, state): if self.recorder is None: raise RuntimeError('recording_forwarder_uninstalled') return self.recorder.seam_callback(event, state) def supplemental(self, state): if self.observer is None: raise RuntimeError('recording_observer_uninstalled') return self.observer.supplemental(state) def bounded_traceback(error): """Retain the innermost 32 frames, without locals, source lines or values.""" frames = [] omitted = 0 tb = error.__traceback__ while tb is not None: code = tb.tb_frame.f_code frames.append({'file': code.co_filename[-512:], 'line': tb.tb_lineno, 'function': code.co_name[:128]}) if len(frames) > 32: del frames[0] omitted += 1 tb = tb.tb_next return {'frames': frames, 'omitted_outer_frames': omitted} def external_failure(phase, error): prefix = {'inference': 'inference', 'identity': 'identity', 'recording': 'recording', 'finalize': 'recording_persistence'}.get(phase, 'infrastructure') if isinstance(error, (KeyboardInterrupt, InterruptedError)): prefix = 'process_signal' context = {'phase': phase, 'type': type(error).__name__, 'message': str(error), 'traceback': bounded_traceback(error)} diagnostic = getattr(error, 'binding_diagnostic', None) if diagnostic is not None: context['binding_diagnostic'] = diagnostic return {'causes': [prefix + '_external_failure'], 'fatal': True, 'context': context} def execute_lifecycle(api, cfg, protocol, indices, output, budget, verify): """One constructor, stable forwards, actual step results, durable P2 transitions. api is an explicit CPU-test seam; CLI always obtains it from real_runtime. There is no plugin/provider flag or production verifier injection path. """ rows, failures = [], [] continuation_allowed = True env = hooks = recorder = None forward = ForwardingEvidence() current = indices[0] phase = 'identity' counts = None try: verify(current) budget.check() with api.hooks(before_step=budget.before_step) as hooks: try: for position, current in enumerate(indices): cfg.seed = protocol.trial(current).seed # sole declared transition phase = 'identity' verify(current) # same used cfg, never normalize here budget.check() phase = 'recording' recorder = api.recorder(protocol, current, output / f'trial-{current:02d}', budget_check=budget.recording, supplemental=forward.supplemental) forward.recorder = recorder stop = failure = None acknowledged = False report = None try: phase = 'initialization' if position == 0: env = api.environment(cfg=cfg, protocol=protocol, trial_index=current, runtime_hooks=hooks, evidence_callback=forward, seam_callback=forward.seam) phase = 'identity' verify(current) # immediate constructor boundary, before reset/action phase = 'initialization' # Enter only after construction, before any published reset snapshot. observers = ExitStack() try: forward.observer = observers.enter_context(api.instrumentation( env, backend_rng=api.backend_rng, generators={})) except BaseException: observers.close() raise observation, _ = env.reset() else: observation, _ = env.begin_trial(protocol, current) stop = env.stop_result if protocol.document['stage'] == 'P2' and current % 2: phase = 'identity' initial = recorder.audit.report().get('initial_pair_state') if (not rows or rows[-1]['trial']['index'] != current - 1 or not isinstance(initial, dict) or not initial or rows[-1].get('initial_pair_state') != initial): raise RuntimeError('identity_pair_initial_state_mismatch') tick = 0 while stop is None: budget.check() phase = 'inference' x = observation['policy'] # raw 46D evidence, not normalized model features action, mean, std = api.infer(x, protocol.trial(current).controller_kind) phase = 'recording' causes = recorder.before_control(tick, x, action, mean=mean, std=std) if causes: raise RuntimeError('Rejected control evidence: ' + repr(causes)) phase = 'step' actual = env.step(action) stop = actual.stop tick += actual.ticks_advanced phase = 'recording' causes = recorder.after_step(actual) if causes: raise RuntimeError('Rejected actual step evidence: ' + repr(causes)) observation = actual.observation except BaseException as error: failure = external_failure(phase, error) failures.append(failure) if env is not None: stop = env.stop_result # never manufacture StopResult/actions/ticks finally: def acknowledge(exact): nonlocal acknowledged env.acknowledge_finalized(exact) acknowledged = True try: phase = 'finalize' report = recorder.finalize(stop=stop, external_failure=failure, acknowledge=acknowledge if failure is None else None) except BaseException as error: failures.append(external_failure('finalize', error)) report = recorder.audit.report() report.update(status='persistence_failed', functional_passed=False) # Preserve partial stream; close it without claiming durability. try: recorder.stream.close() except BaseException as close_error: failures.append(external_failure('finalize', close_error)) rows.append(report) if failure is not None or failures or stop is None or stop.fatal or not acknowledged: continuation_allowed = False break recorder = None finally: phase = 'cleanup' try: if 'observers' in locals(): observers.close() finally: if env is not None: env.close() # hooks still installed, cleanup steps cannot evade counting except BaseException as error: failures.append(external_failure(phase, error)) finally: if hooks is not None: counts = {name: getattr(hooks, name) for name in ( 'attempted_steps', 'completed_steps', 'initialization_steps', 'rollout_steps')} try: verify(current) # used actual cfg, after close/guard unwind, including construction failure except BaseException as error: failures.append(external_failure('identity', error)) matrix = api.matrix(protocol, rows) matrix.update(failures=failures, observed_counts=counts, selected_rows=indices, last_trial_index=current, equality_scope='recorded_application_controller_and_enumerated_observable_rng_only', complete_backend_state_equality=False, deterministic_replay=False, continuation_eligible=continuation_allowed and not failures and all( row.get('terminal', {}).get('runtime_stop') is not None and not row['terminal']['runtime_stop']['fatal'] for row in rows) and len(rows) == len(indices), accepted_skill=False, optimizer_steps=0) return matrix def preimport_replicator(): """Only the intended optional package may be absent; broken dependencies fail.""" missing = None try: importlib.import_module(OPTIONAL_REPLICATOR) except ModuleNotFoundError as error: if error.name not in ('omni.replicator', OPTIONAL_REPLICATOR): raise missing = error.name return {'module': OPTIONAL_REPLICATOR, 'available': missing is None, 'missing_module': missing, 'sampling_scope': 'no configured sampling', 'scope_requires_nominal_config_qualification': True, 'complete_rng_state_claim': False} def check_replicator_inventory(inventory): """Availability is bound by exact reviewed module membership, not a grant.""" if (OPTIONAL_REPLICATOR in sys.modules) != (OPTIONAL_REPLICATOR in inventory['modules']): raise RuntimeError('identity_optional_replicator_availability_changed') def stored_config_fields(value): """Binding's dataclass/ClassVar/instance/slot field set, without properties.""" missing = object() def stored(name): item = inspect.getattr_static(value, name, missing) if item is missing: raise ValueError('Unstored dataclass config field: ' + name) if type(item) is MemberDescriptorType: return item.__get__(value, type(value)) if inspect.getattr_static(type(item), '__get__', None) is not None: # Functions/classes are legitimate stored values, not evaluated descriptors. if not (inspect.isfunction(item) or inspect.isclass(item) or inspect.isbuiltin(item)): raise ValueError('Unsupported config field descriptor: ' + name) return item fields = {name: stored(name) for name in inspect.getattr_static(type(value), '__dataclass_fields__')} dictionary = next((vars(cls)['__dict__'] for cls in type(value).__mro__ if '__dict__' in vars(cls)), None) if type(dictionary) is GetSetDescriptorType: fields.update(dictionary.__get__(value, type(value))) elif dictionary is not None: raise ValueError('Unsupported config instance dictionary descriptor') for cls in type(value).__mro__: slots = vars(cls).get('__slots__', ()) for slot in (slots,) if type(slots) is str else slots: if slot not in ('__dict__', '__weakref__') and inspect.getattr_static(value, slot, missing) is not missing: with suppress(AttributeError): # uninitialized slot, as in binding._config_fields fields[slot] = stored(slot) return fields def config_references(config, binding): """Collect exact ResolvableStrings, rejecting invalid graphs before imports.""" references, active = [], set() def visit(value): if binding._resolvable(value): reference = str(value) if reference.count(':') != 1: raise ValueError('Unsupported ResolvableString reference') module, attribute = reference.split(':') if not attribute.isidentifier() or not all(p.isidentifier() for p in module.split('.')): raise ValueError('Unsupported ResolvableString reference') references.append(value) return if value is None or type(value) in (str, bool, int, Path, PosixPath, WindowsPath): return if type(value) is float: if not math.isfinite(value): raise ValueError('Nonfinite effective config') return if inspect.isclass(value) or inspect.isfunction(value) or inspect.isbuiltin(value): return if id(value) in active: raise ValueError('Cyclic effective config') active.add(id(value)) try: if isinstance(value, enum.Enum): children = (value.value,) elif type(value) in (dict, MappingProxyType): if any(type(k) is not str for k in value): raise ValueError('Effective config mapping keys must be strings') children = value.values() elif type(value) in (tuple, list): children = value elif type(value) is slice: children = (value.start, value.stop, value.step) elif type(value) is functools.partial: children = (value.func, value.args, value.keywords) elif dataclasses.is_dataclass(type(value)): children = stored_config_fields(value).values() else: raise ValueError('Unsupported effective config type: ' + type(value).__qualname__) for child in children: visit(child) finally: active.remove(id(value)) visit(config) return references def config_module_path(name, binding): """Only exact file-backed references in the existing bound source namespaces. No find_spec parent imports, prefix wildcard imports, or external inventory inference. Loaded parent search paths must also be the installed directories. """ roots = {'isaaclab': binding.ISAAC_ROOT / 'source/isaaclab/isaaclab', 'isaaclab_physx': binding.ISAAC_ROOT / 'source/isaaclab_physx/isaaclab_physx', 'SkillFactory': ROOT / binding.PACKAGE} parts = name.split('.') if parts[0] not in roots: raise ValueError('Config reference requires explicit external inventory: ' + name) directory = roots[parts[0]] for index in range(len(parts)): if index: directory = directory / parts[index] package = directory / '__init__.py' path = package if package.is_file() else directory.with_suffix('.py') if not path.is_file(): raise ModuleNotFoundError('Missing installed config module: ' + name, name=name) path = binding._file(path) parent_name = '.'.join(parts[:index + 1]) module = sys.modules.get(parent_name) if module is not None: if binding._module_file(parent_name) != path: raise ValueError('Config reference installed source mismatch: ' + parent_name) if path == package and list(vars(module).get('__path__', ())) != [str(directory)]: raise ValueError('Config reference package search path mismatch: ' + parent_name) if index == 0 and module is None: raise ValueError('Config reference requires already imported installed root: ' + parent_name) if index < len(parts) - 1 and path != package: raise ValueError('Config reference parent is not an installed package: ' + parent_name) return path def preimport_config_references(config, binding): """Explicit runner import phase; never calls references or changes their caches. The unchanged encoder/import inventory binds these modules and their source / direct dependency closure later. This report is discovery, not a trust grant. """ references = config_references(config, binding) names = sorted({str(value).split(':')[0] for value in references}) paths = {name: config_module_path(name, binding) for name in names} report = {'references': sorted({str(value) for value in references}), 'modules': {}} for name, expected in paths.items(): importlib.import_module(name) # absence and transitive failures deliberately propagate path = binding._module_file(name) if path != expected or config_module_path(name, binding) != expected: raise ValueError('Config reference imported source mismatch: ' + name) report['modules'][name] = {'path': str(path), 'sha256': binding.sha256(path.read_bytes())} for value in references: binding._reference_target(value, set()) # static named-symbol/cache validation only return report def preforward_video_recorder(cfg, env_module): """Mirror pinned ManagerBasedRLEnv.__init__ forwarding before config binding.""" env_module.verify_installed_stepping(Path('/home/cen/IsaacLab/source/isaaclab/isaaclab/envs')) # EvaluationContactCycleEnv calls super().__init__(cfg), i.e. render_mode=None. if cfg.video_recorder is not None: cfg.video_recorder.env_render_mode = None cfg.video_recorder.eye = tuple(float(x) for x in cfg.viewer.eye) cfg.video_recorder.lookat = tuple(float(x) for x in cfg.viewer.lookat) def preimport_constructor_modules(binding): """Eagerly resolve the finite reviewed constructor set; never hide new imports. Transitive imports remain subject to the unchanged full binding inventory and immediate postconstructor/final equality checks, not an import ignorelist. """ report = {} for name, (filename, expected_hash) in CONSTRUCTOR_IMPORT_PINS.items(): expected = Path(filename) if (config_module_path(name, binding) != expected or binding.sha256(expected.read_bytes()) != expected_hash): raise ValueError('Constructor import source mismatch: ' + name) importlib.import_module(name) if binding._module_file(name) != expected: raise ValueError('Constructor imported source mismatch: ' + name) report[name] = {'path': filename, 'sha256': binding.sha256(expected.read_bytes())} if report[name]['sha256'] != expected_hash: raise ValueError('Constructor import source changed: ' + name) return report # Same-named eager submodule imports bypass this package's lazy function export. PHYSX_EXPORT_SOURCES = { '/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/__init__.pyi': '811bf1b50a42f0c854f13a8c26a9cf1a00e00120132c633c38b58375908f8dc6', '/home/cen/isaacsim-6.0.1/kit/python/lib/python3.12/site-packages/lazy_loader/__init__.py': '9017e7e780ffe7742b8cd25882c22c93acb9ac353f3372b8dcc76acf832acf29', } def physx_constructor_export(binding, *, resolve=False, expected=None): """Resolve once before binding; later checks never repair namespace drift.""" package_name = 'isaaclab_physx.cloner' module_name = package_name + '.physx_replicate' sources = dict(PHYSX_EXPORT_SOURCES) for name in (package_name, module_name): filename, digest = CONSTRUCTOR_IMPORT_PINS[name] if binding._module_file(name) != Path(filename): raise ValueError('identity_physx_export_module_path: ' + name) sources[filename] = digest for filename, digest in sources.items(): if binding.sha256(Path(filename).read_bytes()) != digest: raise ValueError('identity_physx_export_source: ' + filename) package, module = sys.modules[package_name], sys.modules[module_name] target = vars(module).get('physx_replicate') if (not inspect.isfunction(target) or target.__module__ != module_name or target.__qualname__ != 'physx_replicate'): raise ValueError('identity_physx_export_function') binding._source_code(target, Path(CONSTRUCTOR_IMPORT_PINS[module_name][0])) if expected is not None and target is not expected: raise ValueError('identity_physx_export_changed_callable') if resolve: resolver = vars(package).get('__getattr__') loader_path = Path(next(p for p in PHYSX_EXPORT_SOURCES if '/lazy_loader/' in p)) if not inspect.isfunction(resolver) or resolver.__module__ != 'lazy_loader': raise ValueError('identity_physx_export_resolver') binding._source_code(resolver, loader_path) if resolver('physx_replicate') is not target: raise ValueError('identity_physx_export_resolution') if vars(package).get('physx_replicate') is not target: raise ValueError('identity_physx_export_namespace') return target, {'package': package_name, 'symbol': 'physx_replicate', 'module': module_name, 'qualname': target.__qualname__, 'sources': sources} def verify_constructor_export(runtime): _, metadata = physx_constructor_export(runtime.binding, expected=runtime.constructor_export) if metadata != runtime.generator_inventory['constructor_export']: raise ValueError('identity_physx_export_metadata') def real_runtime(args, output): """Actual imported runtime; no environment/sim.step here, including preparation.""" os.chdir(ROOT) from isaaclab.app import AppLauncher app_module = sys.modules[AppLauncher.__module__] if not str(Path(app_module.__file__).resolve()).startswith('/home/cen/IsaacLab/source/'): raise RuntimeError('identity_require_actual_/home/cen/IsaacLab_imports') app = AppLauncher(headless=True, device=args.device, fast_shutdown=True, kit_args=f'--/log/file={Path(args.native_output) / "kit.log"}').app try: inventory = (None if getattr(args, 'discover_inventory', False) else json.loads(Path(args.inventory).read_bytes())) # Execution always uses reviewed eager imports; discovery is preparation-only. for name in (() if inventory is None else inventory['modules']): importlib.import_module(name) modules = {name: importlib.import_module(TASK + name) for name in ( 'evaluation_cfg', 'evaluation_env', 'evaluation_recording', 'evaluation_instrumentation', 'evaluation_rng_scope')} for name in ('evaluation_protocol', 'evaluation_audit', 'evaluation_binding', 'evaluation_budget', 'evaluation_adoption', 'replay'): modules[name] = importlib.import_module(LEG + name) ins = modules['evaluation_instrumentation'] for name in ins.BACKEND_SOURCE_INVENTORY: importlib.import_module(name) # ManagerBasedEnv.seed treats Replicator as optional. Do not enable an # extension, mask transitive defects, or claim its hidden RNG is observed. replicator = preimport_replicator() print(json.dumps({'optional_replicator': replicator}), flush=True) if inventory is not None: check_replicator_inventory(inventory) # Numerical lazy paths also belong in the reviewed inventory. import pytest import torch if torch.device(args.device).type == 'cuda': torch.cuda.init() torch.cuda.get_rng_state(torch.device(args.device)) ins.verify_backend_sources() replay = modules['replay'] if not (ROOT / replay.CANDIDATE).is_file(): raise RuntimeError('identity_real_artifact_test_cannot_skip_missing_candidate') # Required unchanged CPU test in THIS file-backed, correctly imported process. if pytest.main(['-q', '-p', 'no:cacheprovider', REAL_TEST]) != 0: raise RuntimeError('identity_real_artifact_CPU_test_failed') cfg = modules['evaluation_cfg'].EvaluationContactCycleEnvCfg() protocol = modules['evaluation_protocol'].load_protocol(ROOT / args.protocol) cfg.sim.device = args.device cfg.seed = protocol.trial(args.trial_index).seed binding = modules['evaluation_binding'] for name in binding.REQUIRED_RUNTIME: importlib.import_module(name) preforward_video_recorder(cfg, modules['evaluation_env']) constructor_imports = preimport_constructor_modules(binding) constructor_export, export_metadata = physx_constructor_export(binding, resolve=True) config_imports = preimport_config_references(cfg, binding) print(json.dumps({'config_reference_imports': config_imports}), flush=True) binding.normalize_config(cfg) # exactly once, actual config used later scope = modules['evaluation_rng_scope'] inventory_report = scope.qualify_config(cfg) inventory_report['optional_replicator'] = replicator inventory_report['config_reference_imports'] = config_imports inventory_report['constructor_imports'] = constructor_imports inventory_report['constructor_export'] = export_metadata physx_constructor_export(binding, expected=constructor_export) ins.qualify_backend_rng(scope.backend_rng(None)) arguments = dict(root=ROOT, protocol_path=args.protocol, effective_config=cfg, future_sources=FUTURE_SOURCES, declared_overrides={'seed': 'protocol.trial.seed'}, dependency_inventory=inventory, entrypoints=(SCRIPT,)) api = SimpleNamespace( environment=modules['evaluation_env'].EvaluationContactCycleEnv, hooks=modules['evaluation_env'].RuntimeHooks.isaac, recorder=modules['evaluation_recording'].EvaluationRecorder, instrumentation=ins.EvaluationInstrumentation, backend_rng=scope.backend_rng, matrix=modules['evaluation_audit'].matrix_report, ) return SimpleNamespace(app=app, cfg=cfg, protocol=protocol, binding=binding, arguments=arguments, api=api, replay=replay, torch=torch, generator_inventory=inventory_report, constructor_export=constructor_export) except BaseException: import traceback traceback.print_exc() app.close(exit_code=1) # supported fast shutdown must preserve failure raise def runtime_metadata(runtime): torch = runtime.torch device = torch.device(runtime.cfg.sim.device) driver = Path('/proc/driver/nvidia/version') return {'torch': str(torch.__version__), 'numpy': str(importlib.import_module('numpy').__version__), 'warp': str(importlib.import_module('warp').__version__), 'device': str(device), 'device_name': torch.cuda.get_device_name(device) if device.type == 'cuda' else 'CPU', 'torch_cuda': torch.version.cuda, 'driver': driver.read_text() if driver.exists() else 'unavailable', 'generator_inventory': runtime.generator_inventory, 'evidence_scope': importlib.import_module(TASK + 'evaluation_rng_scope').EVIDENCE_SCOPE, 'unobserved_internals': importlib.import_module(TASK + 'evaluation_rng_scope').UNOBSERVED_INTERNALS} class InventoryPreparationError(ValueError): """Unreviewed diagnostic only; partial identities cannot become a manifest.""" def __init__(self, diagnostics): self.report = {'status': 'FAIL', 'inventory_status': 'unreviewed_diagnostic', 'execution_authorized': False, 'environment_constructed': False, 'sim_step_calls': 0, 'invalid_entries': diagnostics, 'note': 'Invalid entries were not hashed or accepted; discovery is incomplete.'} super().__init__('Inventory preparation FAIL: ' + '; '.join( row['entry'] + ': ' + row['error'] for row in diagnostics)) def proposed_native_layouts(roots): """Independent known layouts only for roots with validated file identities.""" layouts = {} if 'torch' in roots: layouts['torch'] = (roots['torch'] / 'lib',) if 'numpy' in roots: layouts['numpy'] = (roots['numpy'] / ('_core' if (roots['numpy'] / '_core').is_dir() else 'core'), roots['numpy'] / 'random', roots['numpy'] / 'linalg', roots['numpy'] / 'fft', roots['numpy'].parent / 'numpy.libs') if 'warp' in roots: layouts['warp'] = (roots['warp'] / 'bin',) return layouts def proposed_inventory(binding, cfg): """Collect independent incompatibilities, never approve a partial inventory. Inspect only valid imported identities. An invalid provider is not traversed; other independent source/module/native entries can still be diagnosed. """ diagnostics = [] def invalid(stage, entry, error): diagnostics.append({'stage': stage, 'entry': entry, 'error': str(error), 'module_observation': binding.module_diagnostic(entry)}) _, names = binding._sources(ROOT, FUTURE_SOURCES, (SCRIPT,)) roles = set(names) binding._encode(cfg, names, set()) source_records = binding._imports(names, ROOT, diagnostics=diagnostics) sources = set(source_records) numerical = binding.NUMERICAL_ROOTS required = binding.required_dependencies(sources, roles) modules = {} for name in sorted(required): try: # Required modules must already be imported by the actual runtime. # Never interpret incidental __module__ metadata as an import name. modules[name] = binding._dependency_module_identity(name) except (ValueError, OSError, ImportError) as error: invalid('dependency-module', name, error) roots = {name: Path(modules[name]['path']).parent for name in numerical if name in modules} layouts = proposed_native_layouts(roots) artifacts = {} def add(path, owner, vendor=False): try: path = binding._file(path) label = str(path) if label in artifacts: raise ValueError('Duplicate proposed native artifact: ' + label) artifacts[label] = {'module': owner, 'path': label, 'sha256': binding.sha256(path.read_bytes()), 'purpose': 'Installed numerical native implementation consumed by ' + owner} if vendor: artifacts[label]['vendor'] = 'NVIDIA' except (ValueError, OSError) as error: invalid('native-artifact', str(path), error) for owner, directories in layouts.items(): for directory in directories: paths = sorted(directory.glob('*.so*')) if not paths: invalid('native-layout', str(directory), 'Unsupported/missing numerical native layout') for path in paths: add(path, owner) if 'torch' in roots and (roots['torch'] / 'lib/libtorch_cuda.so').is_file(): for package in binding.NVIDIA_CUDA_PACKAGES: directory = roots['torch'].parent / 'nvidia' / package / 'lib' paths = sorted(directory.glob('*.so*')) if package in ('cuda_runtime', 'cublas', 'cudnn') and not paths: invalid('native-layout', str(directory), 'Unsupported/missing NVIDIA CUDA native layout') for path in paths: add(path, 'torch', vendor=True) if diagnostics: raise InventoryPreparationError(diagnostics) proposal = {'scope': binding.INVENTORY_SCOPE, 'modules': modules, 'artifacts': artifacts} try: return binding._dependency_inventory(proposal, sources, roles) except (ValueError, OSError, ImportError) as error: invalid('inventory-validation', 'proposal', error) raise InventoryPreparationError(diagnostics) from error def bound_filesystem_inputs(manifest): """Extract immutable file identities, not runtime objects, for external rehash.""" document = json.loads(manifest) files, links = {}, {} def visit(value): if isinstance(value, dict): if {'path', 'target', 'device', 'inode', 'target_sha256'} <= value.keys(): links[value['path']] = dict(value) if isinstance(value.get('path'), str) and isinstance(value.get('sha256'), str): path, digest = value['path'], value['sha256'] if path in files and files[path] != digest: raise ValueError('identity_conflicting_file_hash: ' + path) files[path] = digest for item in value.values(): visit(item) elif isinstance(value, list): for item in value: visit(item) visit(document) files.update(document.get('runtime_layout_pins', {})) # Frozen bundle verification remains the unchanged child's verify_bundle. # Independently inventory every immutable prepared01 file for postexit equality. frozen = ROOT / 'logs/contact_cycle/sim_residual_training_v1/prepared01' for path in frozen.rglob('*'): if path.is_file(): files[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest() return {'files': files, 'links': links, 'frozen_root': str(frozen), 'scope': 'immutable-files-only; no postexit same-object cfg claim'} def verify_filesystem_inputs(evidence): for filename, expected in evidence.get('links', {}).items(): path = Path(filename) stat = path.lstat() target = str(path.readlink()) if (target != expected['target'] or stat.st_dev != expected['device'] or stat.st_ino != expected['inode'] or hashlib.sha256(target.encode()).hexdigest() != expected['target_sha256']): raise ValueError('identity_postexit_link_changed: ' + filename) for filename, expected in evidence['files'].items(): path = Path(filename) if (not path.is_absolute() or path != path.resolve(strict=True) or path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != expected): raise ValueError('identity_postexit_file_changed: ' + filename) frozen = Path(evidence['frozen_root']) expected = {p for p in evidence['files'] if Path(p).is_relative_to(frozen)} if expected != {str(p) for p in frozen.rglob('*') if p.is_file()}: raise ValueError('identity_postexit_frozen_file_set_changed') return {'status': 'PASS', 'files_rehashed': len(evidence['files']), 'scope': evidence['scope']} def prepare(args): discovery = getattr(args, 'discover_inventory', False) if (args.output is None or args.artifact_root is None or (args.inventory is None and not discovery) or (args.inventory is not None and discovery)): raise ValueError('Preparation requires root/output and exactly one of reviewed inventory or discovery') protect_artifact_path(args.output) b = cpu_module('evaluation_budget') protect_artifact_path(args.artifact_root) if not args.output.is_relative_to(args.artifact_root) or args.output == args.artifact_root: raise ValueError('Preparation outputs must belong to aggregate artifact root') b.durable_directory(args.output) native = args.artifact_root / 'native' if native != native.resolve(): raise RuntimeError('storage_native_redirect_symlink') if not native.exists(): b.durable_directory(native) native = native / args.output.name b.durable_directory(native) args.native_output = str(native) def write(path, data): if (b.artifact_size(args.artifact_root, exclude_native=True) + len(data) > b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE): raise RuntimeError('storage_aggregate_cap') b.durable_write(path, data) runtime = None completed = False started = b.boottime() try: runtime = real_runtime(args, args.output) if discovery: try: inventory = proposed_inventory(runtime.binding, runtime.cfg) except InventoryPreparationError as error: error.report['module_observations'] = { name: runtime.binding.module_diagnostic(name) for name, module in tuple(sys.modules.items()) if isinstance(module, ModuleType)} write(args.output / 'inventory.diagnostic.json', b.encode(error.report)) raise write(args.output / 'inventory.proposed.json', b.encode(inventory)) runtime.arguments['dependency_inventory'] = inventory manifest = runtime.binding.prepare_binding(trial_index=args.trial_index, **runtime.arguments) write(args.output / 'binding.json', manifest) write(args.output / 'incidental-observations.json', b.encode( runtime.binding.incidental_observations( set(json.loads(manifest)['imports']) | set(runtime.arguments['dependency_inventory']['modules'])))) metadata = runtime_metadata(runtime) verify_constructor_export(runtime) runtime.binding.verify_binding(manifest, trusted_manifest_sha256=hashlib.sha256(manifest).hexdigest(), trial_index=args.trial_index, **runtime.arguments) filesystem = bound_filesystem_inputs(manifest) verify_filesystem_inputs(filesystem) write(args.output / 'preclose-filesystem.json', b.encode(filesystem)) write(args.output / 'preparation.json', b.encode({ 'storage_scope': b.STORAGE_SCOPE, 'native_hard_cap': False, 'launch_identity': launch_identity(), 'manifest_sha256_for_independent_review': hashlib.sha256(manifest).hexdigest(), 'execution_authorized': False, 'environment_constructed': False, 'sim_step_calls': 0, 'inventory_status': 'unreviewed_proposal' if discovery else 'caller_supplied', 'binding_format': runtime.binding.FORMAT, 'evidence_scope': runtime.binding.PROVENANCE_SCOPE, 'limitations': runtime.binding.PROVENANCE_LIMITATIONS, 'accepted_skill': False, 'prepare_seconds': b.boottime() - started, 'runtime': metadata, 'identity_order': ('actual cfg and inputs verified; evidence durable; then app.close; ' 'parent postexit files/cleanup'), 'same_object_cfg_scope': 'preclose only; no claim after interpreter destruction', 'preclose_identity_verified': True, 'shutdown': 'supported-default-fast', 'final_pass_owner': 'external supervisor after exit', 'note': 'Preparation timing is not an actual backend rollout/storage qualification.'})) completed = True except BaseException as error: write(args.output / 'preparation-failure.json', b.encode({ 'status': 'FAIL', 'error': repr(error), 'failure': external_failure('preparation', error)})) import traceback traceback.print_exc() raise finally: if runtime is not None: runtime.app.close(exit_code=0 if completed else 1) return 0 def ancestor(parent, child): for _ in range(32): fields = Path(f'/proc/{child}/stat').read_text().rsplit(') ', 1)[1].split() child = int(fields[1]) if child == parent: return True if child <= 1: return False return False def worker_payload(fd): """Single-use inherited socket; only a live matching supervisor may hand off.""" with socket.socket(fileno=fd) as channel: channel.settimeout(15) pid, uid, _ = struct.unpack('3i', channel.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) command = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0') if (Path(f'/proc/{pid}/exe').resolve() != Path('/usr/bin/python3.12') or not ancestor(pid, os.getpid())) or uid != os.getuid() or b'--run' not in command or not any( item.decode(errors='replace').endswith(SCRIPT) for item in command): raise RuntimeError('identity_inherited_parent_supervisor_required') channel.sendall(str(os.getpid()).encode() + b'\n') with channel.makefile('rb') as stream: data = stream.read(1 << 20) if len(data) >= 1 << 20: raise RuntimeError('identity_oversized_handoff') return json.loads(data) def run_worker(payload): b = cpu_module('evaluation_budget') if payload.get('launch_identity') != launch_identity(): raise RuntimeError('identity_worker_launch_changed') args = SimpleNamespace(**payload['arguments']) for name in ('artifact_root', 'inventory', 'manifest'): setattr(args, name, Path(getattr(args, name))) reservation = payload['reservation'] index = reservation['index'] output = args.artifact_root / f'launch-{index}' args.trial_index = reservation['grant']['trial_indices'][0] budget = b.LaunchBudget(args.artifact_root, reservation) ledger = json.loads((args.artifact_root / 'budget.json').read_bytes()) if ledger['launches'][-1] != reservation or reservation['status'] != 'reserved': raise RuntimeError('identity_stale_reservation') budget.write(output / 'worker-claim.json', {'pid': os.getpid(), 'reservation': reservation}) handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True)) for s in (signal.SIGTERM, signal.SIGINT)} runtime = None result = {'accepted_skill': False, 'optimizer_steps': 0, 'continuation_eligible': False, 'storage_scope': b.STORAGE_SCOPE, 'observed_counts': None, 'failures': [], 'rows': [{'trial_index': i, 'status': 'not_run'} for i in range(20 if protocol_stage(args.protocol) == 'P2' else 2)]} verify = None try: budget.check() manifest = read_trusted(args.manifest, args.trusted_manifest_sha256) if b.digest(args.inventory) != reservation['grant']['inventory_sha256']: raise RuntimeError('identity_inventory_changed') runtime = real_runtime(args, output) def verify(row): budget.check(cleanup=True) verify_constructor_export(runtime) check_replicator_inventory(runtime.arguments['dependency_inventory']) return runtime.binding.verify_binding( manifest, trusted_manifest_sha256=args.trusted_manifest_sha256, trial_index=row, **runtime.arguments) verify(args.trial_index) frozen, _ = runtime.replay.verify_bundle(ROOT) policy = runtime.replay.load_frozen_policy(ROOT / runtime.replay.CANDIDATE, frozen) policy = policy.to(runtime.cfg.sim.device) def infer(x, kind): if kind not in ('learned_policy', 'analytic_reference'): raise ValueError('identity_unknown_controller') mode = 'learned' if kind == 'learned_policy' else 'reference' return runtime.replay.inference_action(x, mode, policy if mode == 'learned' else None) runtime.api.infer = infer budget.write(output / 'runtime.json', runtime_metadata(runtime)) result = execute_lifecycle(runtime.api, runtime.cfg, runtime.protocol, reservation['grant']['trial_indices'], output, budget, verify) except BaseException as error: result['failures'].append(external_failure('initialization', error)) result['continuation_eligible'] = False finally: if runtime is not None and verify is not None: try: verify(result.get('last_trial_index', args.trial_index)) filesystem = bound_filesystem_inputs(manifest) verify_filesystem_inputs(filesystem) budget.write(output / 'preclose-filesystem.json', filesystem) result['preclose_identity_verified'] = True except BaseException as error: result['failures'].append(external_failure('identity', error)) result['continuation_eligible'] = False result['same_object_cfg_scope'] = 'after env.close, before app.close only' result['final_pass_owner'] = 'external supervisor after exit' result['storage_scope'] = b.STORAGE_SCOPE result['native_hard_cap'] = False persisted = False try: budget.write(output / 'result.json', result) persisted = True finally: for sig, handler in handlers.items(): signal.signal(sig, handler) if runtime is not None: runtime.app.close(exit_code=0 if persisted and result['continuation_eligible'] else 1) return 0 if result['continuation_eligible'] else 1 def process_table(): table = {} for path in Path('/proc').glob('[0-9]*/stat'): try: fields = path.read_text().rsplit(') ', 1)[1].split() table[int(path.parent.name)] = (int(fields[19]), int(fields[1]), int(fields[2]), fields[0]) except (FileNotFoundError, ProcessLookupError): pass return table class OwnedProcesses: """Dedicated sole-spawner supervisor only; not general service containment. Pre-existing child trees are refused. Track birth identities, not names/PIDs; orphan attribution relies on this process spawning no other jobs after start. """ def __init__(self): table = process_table() self.supervisor = (os.getpid(), table[os.getpid()][0]) if any(row[1] == os.getpid() for row in table.values()): raise RuntimeError('process_dedicated_supervisor_has_preexisting_children') self.preexisting = {(pid, row[0]) for pid, row in table.items()} self.known = set() self.root = None self.unknown = False def attach(self, child): row = process_table().get(child.pid) if row is None: raise RuntimeError('process_spawn_identity_unavailable') self.root = (child.pid, row[0]) self.known.add(self.root) def discover(self): table = process_table() if table.get(self.supervisor[0], (None,))[0] != self.supervisor[1]: self.unknown = True return {} # Retain exclusions for observed descendants of unrelated baseline # processes as well, even if their parent later exits/adopts here. changed = True while changed: changed = False for pid, row in table.items(): identity = (pid, row[0]) parent = table.get(row[1]) if (identity not in self.preexisting and identity not in self.known and parent is not None and (row[1], parent[0]) != self.supervisor and (row[1], parent[0]) in self.preexisting): self.preexisting.add(identity) changed = True if any(row[1] == self.supervisor[0] and (pid, row[0]) in self.preexisting for pid, row in table.items()): self.unknown = True # unexpected unrelated adoption violates dedicated ownership active = {pid: row for pid, row in table.items() if (pid, row[0]) in self.known} changed = True while changed: changed = False for pid, row in table.items(): identity = (pid, row[0]) if pid in active or identity in self.preexisting: continue # Direct children after attach are adopted owned orphans under # the enforced dedicated, sole-spawner architecture. if (self.root is not None or self.unknown) and (row[1] == self.supervisor[0] or row[1] in active): self.known.add(identity) active[pid] = row changed = True return active def signal(self, pid, row, sig): # pidfd pins the process during signal delivery; recheck starttime after # opening to close the discovery -> open PID-reuse window. with suppress(ProcessLookupError, FileNotFoundError): fd = os.pidfd_open(pid) try: current = process_table().get(pid) if current is not None and current[0] == row[0]: signal.pidfd_send_signal(fd, sig) finally: os.close(fd) def reap(self, child): child.poll() for pid, row in self.discover().items(): if pid != child.pid and row[1] == self.supervisor[0] and row[3] == 'Z': # An unreaped child cannot reuse its PID. Never reap unrelated jobs. with suppress(ChildProcessError, ProcessLookupError): os.waitpid(pid, os.WNOHANG) def cleanup_group(child, deadline, b, owned, pump=lambda: None): sent = set() term_until = min(deadline - 10, b.boottime() + 110) while b.boottime() < deadline: sig = signal.SIGTERM if b.boottime() < term_until else signal.SIGKILL for pid, row in owned.discover().items(): key = (pid, row[0], sig) if row[3] != 'Z' and key not in sent: owned.signal(pid, row, sig) sent.add(key) owned.reap(child) pump() # keep draining even when handshake failed or descendants hold pipe if not owned.discover(): return not owned.unknown time.sleep(.05) owned.reap(child) pump() return not owned.discover() and not owned.unknown class StdoutCapture: """Nonblocking capture from spawn through cleanup; overflow drains/discards. Each of three launches has a fixed 1 MiB allowance withheld from all other writers. This avoids a check/write race with the worker's pending trace bytes. The 16 MiB cleanup reserve is separate and never consumed by stdout. """ def __init__(self, pipe, path, b): self.pipe, self.b = pipe, b self.log = Path(path).open('xb', buffering=0) # noqa: SIM115 - owned until cleanup close try: os.set_blocking(pipe.fileno(), False) except BaseException: try: self.log.close() finally: pipe.close() raise self.written = self.discarded = 0 self.eof = False self.error = None def pump(self): # Bounded work per turn, even for an unlimited producer. for _ in range(16): try: block = os.read(self.pipe.fileno(), 65536) except BlockingIOError: break if not block: self.eof = True break retained = block[:max(0, self.b.STDOUT_BYTES - self.written)] try: if retained and self.error is None: if self.log.write(retained) != len(retained): raise OSError('storage_short_stdout_write') self.written += len(retained) except OSError as error: self.error = error self.discarded += len(block) - (len(retained) if self.error is None else 0) return self.discarded def check(self): self.pump() if self.error is not None: raise self.error if self.discarded: raise RuntimeError('storage_stdout_capture_overflow') def close(self): try: self.pump() os.fsync(self.log.fileno()) finally: self.log.close() self.pipe.close() def exchange_worker(child, channel, payload, budget, capture, owned, monitor, b): deadline = min(b.boottime() + 30, budget.reservation['deadline_boottime'] - b.CLEANUP_SECONDS) channel.setblocking(False) hello = b'' while b'\n' not in hello: capture.check() budget.check() monitor.check() owned.discover() if b.boottime() >= deadline: raise RuntimeError('identity_worker_handshake_timeout') try: block = channel.recv(64 - len(hello)) if not block: raise RuntimeError('identity_worker_handshake_eof') hello += block except BlockingIOError: if child.poll() is not None: capture.pump() raise RuntimeError('identity_worker_exited_without_handshake') time.sleep(.01) if len(hello) >= 64: raise RuntimeError('identity_worker_handshake_oversized') worker_pid = int(hello.strip()) if (os.getpgid(worker_pid) != child.pid or worker_pid not in owned.discover() or not ancestor(os.getpid(), worker_pid)): raise RuntimeError('identity_foreign_worker_handshake') channel.settimeout(max(.01, deadline - b.boottime())) channel.sendall(b.encode(payload)) channel.shutdown(socket.SHUT_WR) channel.close() def supervise(args): b = cpu_module('evaluation_budget') policy = storage_policy_check(args, b) if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'): raise RuntimeError('process_identity_safe_pidfd_supervisor_required') grant = grant_check(json.loads(read_trusted(args.grant, args.trusted_grant_sha256)), args, b) profile = resource_profile(args.protocol) slots, _, _ = b.profile_limits(profile) if policy['not_after_boottime'] < max(grant['not_after_boottime'], b.boottime() + slots[grant['launch_index']][2] + (b.CLEANUP_SECONDS if profile is not None else 0)): raise RuntimeError('storage_policy_expires_before_grant') owned = OwnedProcesses() # refuse pre-existing jobs before any reservation monitor = NativeMonitor(policy, b) environment = dict(os.environ, PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', PYTHONDONTWRITEBYTECODE='1') launch_command([], environment) # qualify installed launch path before debit ledger = b.BudgetLedger(args.artifact_root, grant['campaign_id'], cap_bytes=policy['limit_bytes'], resource_profile=profile) child = reservation = channels = capture = None clean = eligible = False counts = None errors = [] handlers = {} monitor_failed = False def cleanup_pump(): nonlocal monitor_failed if capture is not None: capture.pump() try: monitor.check() except Exception as error: if not monitor_failed: errors.append(external_failure('supervisor', error)) monitor_failed = True # keep signaling/reaping even after native stop try: reservation = ledger.reserve(grant, enforcement_deadline=policy['not_after_boottime']) output = args.artifact_root / f'launch-{reservation["index"]}' budget = b.LaunchBudget(args.artifact_root, reservation) b.durable_directory(output) handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True)) for s in (signal.SIGTERM, signal.SIGINT)} budget.write(output / 'reservation.json', reservation) if profile is not None: budget.write(output / 'adopted-p1-evidence.json', cpu_module('evaluation_adoption').verify_adoption( args.artifact_root, grant['p1_adoption_sha256'])) budget.write(output / 'storage-policy.json', monitor.report()) budget.write(output / 'launch-identity.json', launch_identity()) budget.write(output / 'supervisor-identity.json', { 'pid': owned.supervisor[0], 'starttime': owned.supervisor[1], 'dedicated_sole_spawner': True, 'preexisting_children': []}) for name, path in (('manifest.json', args.manifest), ('inventory.json', args.inventory), ('grant.json', args.grant)): data = Path(path).read_bytes() budget.check(pending=len(data)) b.durable_write(output / name, data) if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: raise RuntimeError('process_subreaper_setup_failed') channels = socket.socketpair() parent, child_channel = channels child_channel.set_inheritable(True) native = args.artifact_root / 'native' if native != native.resolve(): raise RuntimeError('storage_native_redirect_symlink') if not native.exists(): b.durable_directory(native) native = native / f'launch-{reservation["index"]}' b.durable_directory(native) redirects = native_redirects(native) for directory in redirects.values(): b.durable_directory(Path(directory)) environment.update(redirects) command = launch_command([str(ROOT / SCRIPT), '--_worker-fd', str(child_channel.fileno())], environment) budget.check() child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True, pass_fds=(child_channel.fileno(),)) try: owned.attach(child) # first operation after spawn; capture setup can fail except BaseException: # Sole-spawner discovery must still collect children if birth lookup # failed; unknown ownership can never be reported as clean. owned.unknown = True raise child_channel.close() capture = StdoutCapture(child.stdout, output / 'stdout.log', b) arguments = {'protocol': args.protocol, 'device': args.device, 'artifact_root': str(args.artifact_root), 'native_output': str(native), 'inventory': str(output / 'inventory.json'), 'manifest': str(output / 'manifest.json'), 'trusted_manifest_sha256': args.trusted_manifest_sha256} exchange_worker(child, parent, {'arguments': arguments, 'reservation': reservation, 'launch_identity': launch_identity()}, budget, capture, owned, monitor, b) while not capture.eof or child.poll() is None: capture.check() budget.check() monitor.check() owned.discover() time.sleep(.01) child.wait(timeout=max(.1, reservation['deadline_boottime'] - b.boottime() - 120)) if (output / 'result.json').is_file(): result = json.loads((output / 'result.json').read_bytes()) counts = result['observed_counts'] postexit = verify_filesystem_inputs(json.loads((output / 'preclose-filesystem.json').read_bytes())) budget.write(output / 'postexit-filesystem.json', postexit) eligible = (child.returncode == 0 and result['continuation_eligible'] and result.get('preclose_identity_verified') is True) except BaseException as error: errors.append(external_failure('supervisor', error)) finally: try: try: if channels: for channel in channels: channel.close() if child is not None and capture is None and child.stdout is not None: child.stdout.close() # constructor/attach failure left no capture owner finally: if child is not None: clean = cleanup_group(child, reservation['deadline_boottime'], b, owned, cleanup_pump) if capture is not None: capture.close() if capture.discarded or capture.error: eligible = False errors.append(external_failure('recording', RuntimeError('storage_stdout_capture_incomplete'))) if reservation is not None: if profile is not None: try: adoption = cpu_module('evaluation_adoption').verify_adoption( args.artifact_root, grant['p1_adoption_sha256']) budget.write(output / 'postexit-adoption.json', { 'p1_adoption_sha256': grant['p1_adoption_sha256'], 'status': 'PASS', 'rows': [{'trial': row['trial'], 'first_failure': row['first_failure'], 'functional_passed': False, 'accepted_skill': False} for row in adoption['rows']], 'historical_reserved_calls': adoption['historical_reserved_calls'], 'local_reserved_calls': 91000, 'cumulative_reserved_calls': adoption['historical_reserved_calls'] + 91000}) except BaseException as error: eligible = False errors.append(external_failure('identity', error)) final_size = b.artifact_size(args.artifact_root, exclude_native=True) eligible = eligible and not errors and final_size < policy['limit_bytes'] - b.HEADROOM_BYTES budget.write(output / 'supervisor.json', { 'cleanup_clean': clean, 'errors': errors, 'returncode': None if child is None else child.returncode, 'artifact_bytes_before_finalization': final_size, 'continuation_eligible': eligible and clean, 'storage': monitor.report(), 'stdout_discarded_bytes': capture.discarded if capture else 0, 'stdout_eof_observed': capture.eof if capture else False, 'owned_process_identities': sorted(owned.known), 'ownership_unknown': owned.unknown, 'reserved_physical_calls': reservation['reserved_physical_calls'], 'observed_counts': counts}) ledger.finish(counts=counts, clean=clean, eligible=eligible) finally: ledger.close() for sig, handler in handlers.items(): signal.signal(sig, handler) return 0 if eligible and clean else 1 def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) modes = parser.add_mutually_exclusive_group(required=True) modes.add_argument('--check-only', action='store_true') modes.add_argument('--prepare-binding', action='store_true') modes.add_argument('--run', action='store_true') modes.add_argument('--_worker-fd', type=int, help=argparse.SUPPRESS) parser.add_argument('--protocol', choices=PROTOCOLS, default=PROTOCOLS[0]) parser.add_argument('--trial-index', type=int, default=0) parser.add_argument('--device', default='cuda:0') parser.add_argument('--inventory', type=Path) parser.add_argument('--discover-inventory', action='store_true', help='Preparation only: propose actual imported inventory; never grants execution trust') parser.add_argument('--output', type=Path) parser.add_argument('--artifact-root', type=Path) parser.add_argument('--manifest', type=Path) parser.add_argument('--trusted-manifest-sha256') parser.add_argument('--grant', type=Path) parser.add_argument('--trusted-grant-sha256') parser.add_argument('--storage-policy', type=Path) parser.add_argument('--trusted-storage-policy-sha256') args = parser.parse_args(argv) if args.discover_inventory and (not args.prepare_binding or args.inventory is not None): parser.error('--discover-inventory requires preparation only, without --inventory') if args._worker_fd is not None: return run_worker(worker_payload(args._worker_fd)) if args.check_only: protocol = cpu_module('evaluation_protocol').load_protocol(ROOT / args.protocol) protocol.trial(args.trial_index) print(json.dumps({'scope': 'CPU protocol/source planning only; no binding, authorization or physical pass', 'stage': protocol.document['stage'], 'rows': [asdict(t) for t in protocol.trials], 'future_sources': FUTURE_SOURCES, 'resource_profile': resource_profile(args.protocol), 'reserved_physical_calls': [slot[0] for slot in cpu_module('evaluation_budget').profile_limits(resource_profile(args.protocol))[0]], 'actual_backend_imports_and_performance': 'unqualified'})) return 0 if args.prepare_binding: return prepare(args) if any(getattr(args, name) is None for name in ('inventory', 'artifact_root', 'manifest', 'trusted_manifest_sha256', 'grant', 'trusted_grant_sha256')): parser.error('--run requires reviewed inventory, artifact root, manifest+trusted digest, ' 'fresh grant+trusted digest') return supervise(args) if __name__ == '__main__': raise SystemExit(main())