Files
SkillFactory/scripts/evaluate_cycle_sim_residual.py
T
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

1948 lines
101 KiB
Python

#!/usr/bin/env python3
"""Prospective P1/P2 evaluator. CPU --check-only; Kit --prepare-binding; gated --run.
No mode grants its own permission. --run is a Linux supervisor, not a direct Kit
invocation. It consumes a fresh independently trusted parent grant and binding.
Never use an old replay grant. See logs/contact_cycle/evaluation_runner_v1/README.md
and docs/contact_cycle_native_monitor_v3.md (superseding native coverage policy).
"""
from __future__ import annotations
import argparse
import ctypes
import dataclasses
import enum
import errno
import functools
import hashlib
import importlib
import importlib.util
import inspect
import json
import math
import os
import shlex
import signal
import socket
import stat
import struct
import subprocess
import sys
import time
from contextlib import ExitStack, suppress
from dataclasses import asdict
from pathlib import Path, PosixPath, WindowsPath
from types import GetSetDescriptorType, MappingProxyType, MemberDescriptorType, ModuleType, SimpleNamespace
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = 'scripts/evaluate_cycle_sim_residual.py'
PACKAGE = 'source/SkillFactory/SkillFactory/'
TASK = 'SkillFactory.tasks.manager_based.contact_cycle_sim.'
LEG = 'SkillFactory.legs.contact_cycle_sim.'
PROTOCOLS = ('configs/contact_cycle/continuous_eval_p1_v1.json',
'configs/contact_cycle/bounded_diagnostic_p2_v1.json',
'configs/contact_cycle/bounded_diagnostic_p2_extended_v1.json')
FUTURE_SOURCES = {
'runner': [SCRIPT, PACKAGE + 'legs/contact_cycle_sim/evaluation_budget.py'],
'instrumentation': [PACKAGE + 'tasks/manager_based/contact_cycle_sim/' + name + '.py'
for name in ('evaluation_instrumentation', 'evaluation_rng_scope')],
}
REAL_TEST = 'tests/test_contact_cycle_sim_replay.py::test_actual_frozen_candidate_cpu_only_when_artifacts_present'
LAUNCHER = '/home/cen/IsaacLab/isaaclab.sh'
OPTIONAL_REPLICATOR = 'omni.replicator.core'
# Exact constructor-late imports evidenced in p1_first_failure_diagnosis.md.
# Full source identities are checked before import and retained in the binding.
CONSTRUCTOR_IMPORT_PINS = {
'isaaclab.cloner.cloner_cfg': (
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cloner/cloner_cfg.py',
'f48e268609e8e5e91e30b8e6d364cc76579aeb497d69f963cd45975152a282ca'),
'isaaclab.cloner.cloner_strategies': (
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cloner/cloner_strategies.py',
'ea5943bb5e17df09b792dcf7de436f7481daed8768dc2d8573c6eb45464481d8'),
'isaaclab.sim.schemas.schemas_actuators': (
'/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/schemas/schemas_actuators.py',
'3defa9a6ff41d39e797c6d4a1dfee031913fdaf64ff7f99305a844f4ddf47f77'),
'isaaclab.sim.spawners.wrappers': (
'/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/spawners/wrappers/__init__.py',
'431dfebf1f14464b5479fe618720d913fdefb8e1d613a0c54f45dcddb671cce5'),
'isaaclab.sim.spawners.wrappers.wrappers_cfg': (
'/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/spawners/wrappers/wrappers_cfg.py',
'ce073f03584f9cab400735fce1af92b261b0e9217592c7ce79c1b8f541399574'),
'isaaclab.sim.views.usd_frame_view': (
'/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/views/usd_frame_view.py',
'6c186fe2991a608323f12f40fb67516dd629079eabe4f6e525c198e0452a59d9'),
'isaaclab.utils.warp.fabric': (
'/home/cen/IsaacLab/source/isaaclab/isaaclab/utils/warp/fabric.py',
'3756853121fbd0fb88b01f07c521927e9cdc1ccd96ceb40b6a603bef29e56c94'),
'isaaclab_physx.assets.surface_gripper': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/assets/surface_gripper/__init__.py',
'42992defc933ea096bac4666c574bfd32ace19fb3886f6e41b5d5498e06da2ef'),
'isaaclab_physx.assets.surface_gripper.surface_gripper_cfg': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/assets/surface_gripper/surface_gripper_cfg.py',
'c71846dad68db328c6faa9ac7dfe37488502e9477f56de0961ab3648f273b14a'),
'isaaclab_physx.cloner': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/__init__.py',
'13e3966a049c8a6feeee9a8018162ddce6667f9cbdc0d7350644b2176a191def'),
'isaaclab_physx.cloner.physx_replicate': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/physx_replicate.py',
'a0e7eb7f47e45ce27351002a0ef6c137ddb3c1c4a20a5efcbbd389c3ff5a426c'),
'isaaclab_physx.sensors': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/__init__.py',
'52dc0bed9f1cc96b938b9a6aae4f2f48076006fce0663c2e8ad017488f8098a4'),
'isaaclab_physx.sensors.contact_sensor': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/__init__.py',
'2f81a5de0c1a2a264fc18832fbfe33e3cb49a41a7a246b0de4471437dcbf3dfd'),
'isaaclab_physx.sensors.contact_sensor.contact_sensor': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py',
'09507db17049bcadd6a48f5992bc20f79b019356c13f1ba3ef1527b2e4188b2c'),
'isaaclab_physx.sensors.contact_sensor.contact_sensor_data': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor_data.py',
'711c461be5c696a7a99f0a8fd2a010cd9a5a47e1b8bcfdf493e5ac8af8b65e3d'),
'isaaclab_physx.sensors.contact_sensor.kernels': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/kernels.py',
'e3e9d678336a704a9ba58fe7d31fe8a3a577f66f357c921a2c312c2edb8fa6e3'),
'isaaclab_physx.sensors.joint_wrench': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/__init__.py',
'308cb956e92e065c05b66ff0ec0c139dc3bc35380a09d58191af13acf5c14f36'),
'isaaclab_physx.sensors.joint_wrench.joint_wrench_sensor': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/joint_wrench_sensor.py',
'eff63d6f6e8c9958651280889c1797c9cbf2a14fb1b35a1f4e39cb208399ee5f'),
'isaaclab_physx.sensors.joint_wrench.joint_wrench_sensor_data': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/joint_wrench_sensor_data.py',
'32b168096dc384587e2962b8e09caccba1d76bb8e40459073b262843b6511937'),
'isaaclab_physx.sensors.joint_wrench.kernels': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/kernels.py',
'a71e33ae91ab7bef9d585d7ca8137028a3dfe5b169dabec4b9b763f8497d25a0'),
'isaaclab_physx.sensors.kernels': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/kernels.py',
'297f79677d58d7c5ed38cffc7ba19480bc3236a331a64700a54d57f34a5b8b56'),
'isaaclab_physx.sim.views': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sim/views/__init__.py',
'ac57ce5aafa903fe42868a410461b9db8fc8a150ed458f9b8bfe80772a94f97c'),
'isaaclab_physx.sim.views.fabric_frame_view': (
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sim/views/fabric_frame_view.py',
'7d9d199be7dc8b9087c60f9f89567fb0463fd0269d20b0aa0e20ae7f8ac1d329'),
}
# One reviewed installed shell/CLI layout only; the bridge changes subprocess.run
# to exec, not the inherited socket authentication or the runtime import tree.
LAUNCH_PINS = {
'/home/cen/IsaacLab/isaaclab.sh':
'c3a6f3111012aa39c5f568012da1ef2cde52a80cbc2277bc59ce295895d7bc68',
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/__init__.py':
'8886104cee16eb41c8c7dccbe2955cf8e4c72677f49423bf586f4eb4615a96af',
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/envs.py':
'92002ebc7301f559cd17dcbf13a749006c1353aabf9f1247e57a9438a85acecd',
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/format.py':
'bc575ef81dff01a4c8a7b11496515ac34fa9bbdf14613978e84fe37449cd64cc',
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/install.py':
'b285da1f19beada43c1a4ad9aa7e3b432a0042eb0074c7c89ccc7aa0d3c46b31',
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/misc.py':
'b608ffecbae98c810024dc361852325d09197a9f9a355b270bd60c410a459747',
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/utils.py':
'9608423230d159234e740a8000fd8206a4c980311e6517f97d785eb58f68fc97',
'/home/cen/IsaacLab/source/isaaclab/isaaclab/__init__.py':
'24e50c4e57b9be1c55aa949f1b8dd416d95e6002e12941abe33a7a6eb16f697b',
'/home/cen/isaacsim-6.0.1/python.sh':
'37f7e3717f1fce1800d0b15b55fcd754c909978bd2455b9d6de3d22d77a92bc3',
'/home/cen/isaacsim-6.0.1/setup_python_env.sh':
'0f73e6dac65efa1a28f994939f4734859d2c39eb236c658bfad9fab70c8882e6',
'/home/cen/isaacsim-6.0.1/kit/python/bin/python3':
'f3300e753c44afd7860b3f26ccf7c3d98c323b4fc90e7ce2fccae2cd49efd09c',
'/home/cen/isaacsim-6.0.1/kit/libcarb.so':
'aef0445bb75bd3da04acfabf6d067fb9e40b5c3d17f07dd5369bd63c75489c4f',
'/usr/bin/python3.12':
'e50d468e8b0adfb05733f5b87b3cff34829c4a8c1aea50c865aa8bdfe4bb150f',
'/usr/bin/bash':
'bc5945feb8bd26203ebfafea5ce1878bb2e32cb8fb50ab7ae395cfb1e1aaaef1',
}
RESOLVED_PYTHON = '/home/cen/isaacsim-6.0.1/kit/python/bin/python3'
EXEC_BRIDGE = """import os, sys
from pathlib import Path
from isaaclab.cli.utils import extract_python_exe
resolved = extract_python_exe()
if Path(resolved).resolve() != Path('/home/cen/isaacsim-6.0.1/python.sh'):
raise RuntimeError('identity_unreviewed_resolved_interpreter')
os.execv(resolved, [resolved, *sys.argv[1:]])
"""
def launch_identity():
b = cpu_module('evaluation_budget')
for path, expected in LAUNCH_PINS.items():
if b.digest(path) != expected:
raise RuntimeError('identity_launch_source_changed: ' + path)
# This layout has no optional setup_conda_env.sh or docker startup helper.
sim = Path('/home/cen/IsaacLab/_isaac_sim')
if (sim.resolve() != Path('/home/cen/isaacsim-6.0.1')
or (sim / 'setup_conda_env.sh').exists() or (sim / 'vulkan_check.sh').exists()
or any(Path('/home/cen/IsaacLab', name).exists() for name in ('env_isaaclab', '.venv'))):
raise RuntimeError('identity_unreviewed_launch_layout')
return {'format': 'skillfactory.fd-exec-launch.v1', 'sources': LAUNCH_PINS,
'bridge_sha256': hashlib.sha256(EXEC_BRIDGE.encode()).hexdigest(),
'runner_sha256': b.digest(ROOT / SCRIPT), 'interpreter': RESOLVED_PYTHON}
def launch_command(arguments, environment):
launch_identity()
if any(environment.get(k) for k in ('VIRTUAL_ENV', 'CONDA_PREFIX', 'PYTHONEXE', 'PYTHONHOME', 'args')):
raise RuntimeError('identity_launch_requires_reviewed_no_virtualenv_layout')
text = Path(LAUNCHER).read_text()
text = text.replace('export ISAACLAB_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"',
'export ISAACLAB_PATH=/home/cen/IsaacLab')
old = 'exec "$python_exe" -c "from isaaclab.cli import cli; cli()" "$@"'
if text.count(old) != 1:
raise RuntimeError('identity_launch_exec_boundary_changed')
text = text.replace(old, 'exec "$python_exe" -c ' + shlex.quote(EXEC_BRIDGE) + ' "$@"')
return ['/usr/bin/bash', '-c', text, LAUNCHER, *arguments]
def cpu_module(name):
"""Load pure stdlib source without importing Isaac registration before Kit.
Budget has its real file-backed module identity (no fake parent packages).
CPU-only protocol planning uses a private identity, never a binding snapshot.
"""
if name == 'evaluation_adoption':
cpu_module('evaluation_budget')
full = LEG + name if name in ('evaluation_budget', 'evaluation_adoption') else '_evaluation_cpu_' + name
if full not in sys.modules:
path = ROOT / PACKAGE / 'legs/contact_cycle_sim' / (name + '.py')
spec = importlib.util.spec_from_file_location(full, path)
module = importlib.util.module_from_spec(spec)
sys.modules[full] = module
spec.loader.exec_module(module)
return sys.modules[full]
def protect_artifact_path(path):
path = Path(path)
allowed = ROOT / 'logs/contact_cycle/evaluation_runs_v1'
if (not path.is_absolute() or path != path.resolve()
or (path.is_relative_to(ROOT) and not path.is_relative_to(allowed))):
raise ValueError('Use new evaluation_runs_v1 campaign paths or canonical paths outside repository; '
'historic evidence/source/release trees are protected')
def read_trusted(path, expected):
payload = Path(path).read_bytes()
if (len(expected or '') != 64 or any(c not in '0123456789abcdef' for c in expected)
or hashlib.sha256(payload).hexdigest() != expected):
raise ValueError('identity_independently_trusted_digest_required: ' + str(path))
return payload
def resource_profile(protocol_path):
if protocol_path not in PROTOCOLS:
raise ValueError('Unknown fixed protocol path')
protocol = cpu_module('evaluation_protocol').load_protocol(ROOT / protocol_path)
return protocol.document.get('resource_profile')
def protocol_stage(protocol_path):
"""Fixed-path stage lookup, safe before real runtime binding.
Never load a private CPU planning module into the Kit worker. Full protocol
validation remains with the supervisor and the real runtime module.
"""
if protocol_path not in PROTOCOLS:
raise ValueError('Unknown fixed protocol path')
return 'P1' if protocol_path == PROTOCOLS[0] else 'P2'
def grant_check(grant, args, b):
extended = resource_profile(args.protocol) == b.EXTENDED_PROFILE
fields = {'format', 'campaign_id', 'launch_index', 'stage', 'trial_indices', 'manifest_sha256',
'protocol_sha256', 'inventory_sha256', 'artifact_root', 'boot_id',
'not_after_boottime', 'parent_review', 'p1_evidence_sha256', 'storage_policy_sha256',
'storage_scope', 'launch_identity_sha256'}
if extended:
fields = (fields - {'p1_evidence_sha256'}) | {'resource_profile', 'p1_adoption_sha256'}
if not isinstance(grant, dict) or set(grant) != fields:
raise ValueError('Explicit fresh parent grant schema required')
index = grant['launch_index']
if type(index) is not int or index not in ((0,) if extended else (0, 1, 2)):
raise ValueError('Parent grant launch slot')
expected_stage = 'P2' if extended or index == 2 else 'P1'
rows = list(range(20)) if expected_stage == 'P2' else [index]
if (grant['format'] != (b.EXTENDED_GRANT_FORMAT if extended else 'skillfactory.evaluation-parent-grant.v2')
or not isinstance(grant['campaign_id'], str) or not grant['campaign_id']
or not isinstance(grant['parent_review'], str) or not grant['parent_review'].strip()
or grant['stage'] != expected_stage or grant['trial_indices'] != rows
or grant['manifest_sha256'] != args.trusted_manifest_sha256
or grant['protocol_sha256'] != b.digest(ROOT / args.protocol)
or grant['inventory_sha256'] != b.digest(args.inventory)
or grant['storage_policy_sha256'] != args.trusted_storage_policy_sha256
or grant['storage_scope'] != b.STORAGE_SCOPE
or grant['launch_identity_sha256'] != hashlib.sha256(b.encode(launch_identity())).hexdigest()
or grant['artifact_root'] != str(args.artifact_root)
or grant['boot_id'] != b.boot_id()
or args.protocol != PROTOCOLS[2 if extended else index == 2]
or type(grant['not_after_boottime']) not in (int, float)
or not b.boottime() < grant['not_after_boottime'] <= b.boottime() + 3600
or (not extended and index < 2 and grant['p1_evidence_sha256'] != [])
or (not extended and index == 2 and (not isinstance(grant['p1_evidence_sha256'], list)
or len(grant['p1_evidence_sha256']) != 2))):
raise ValueError('Parent grant identity/order/expiry/review mismatch')
if extended:
if grant['resource_profile'] != b.EXTENDED_PROFILE or args.artifact_root not in (b.EXTENDED_ROOT, b.SECOND_EXTENDED_ROOT):
raise ValueError('Parent grant fixed extended profile/root mismatch')
adoption = cpu_module('evaluation_adoption').verify_adoption(args.artifact_root, grant['p1_adoption_sha256'])
if adoption['source_boot_id'] != b.boot_id():
raise ValueError('Parent grant adoption same-boot mismatch')
read_trusted(args.manifest, args.trusted_manifest_sha256)
return grant
def mount_identity(path):
"""Read-only mount facts, NOT evidence that a quota or confinement is effective."""
matches = []
for line in Path('/proc/self/mountinfo').read_text().splitlines():
fields = line.split()
mount = Path(fields[4].replace('\\040', ' '))
if path == mount or mount in path.parents:
matches.append((len(mount.parts), line))
if not matches:
raise RuntimeError('storage_mount_identity_unavailable')
return max(matches)[1]
STORAGE_APPROVAL = 'logs/contact_cycle/evaluation_runner_recovery_v1/USER_STORAGE_SCOPE_AMENDMENT.md'
STORAGE_APPROVAL_SHA256 = 'e4fac0d36799c9beb67af7687a390a6b5e8390bde1d4b840a6e26c309578c8db'
NATIVE_GAPS = ('sampled logical sizes, not a quota; writes can overshoot between checks',
'unlisted/global cache writers, deleted-open files, symlink targets and native allocations '
'have unknown bytes/growth; only free-space floors on listed filesystems cover external writes',
'shared native cache changes include unrelated jobs; growth can cause conservative stops',
'stable non-runtime-owned inaccessible shared subtrees are explicit permission gaps; '
'their bytes and growth are unknown; coverage is incomplete',
'generic HOME/.cache and /tmp are not recursively scanned; unlisted or hidden mounts are uncovered',
'installed data and kit/data/cache/logs are conservative portable-token candidates; '
'actual token destinations require runtime confirmation; missing candidates are not created')
SIM_ROOT = Path('/home/cen/isaacsim-6.0.1')
def native_paths(root):
# SimulationApp --portable + kit-core.json data/log writers; token resolution
# remains unqualified. Never substitute a generic home/cache/tmp tree.
return sorted({str(root / 'native'), *shared_native_paths(),
*[str(SIM_ROOT / name) for name in ('data', 'kit/data', 'kit/logs')]})
def shared_native_paths():
# Installed clear_caches.sh names these three ov caches explicitly.
return sorted({*[str(Path.home() / '.cache/ov' / name)
for name in ('shaders', 'texturecache', 'Kit/107.3')], str(SIM_ROOT / 'kit/cache')})
def free_space_paths(root):
# Other native roots also observe their own filesystem during their scan.
return sorted({str(root), '/tmp', str(Path.home()), str(SIM_ROOT)})
def native_redirects(native, *, preparation=False):
redirects = {'HOME': str(native / 'home'), 'XDG_CACHE_HOME': str(native / 'cache'),
'TMPDIR': str(native / 'tmp')}
if preparation:
redirects.update(XDG_CONFIG_HOME=str(native / 'config'), XDG_DATA_HOME=str(native / 'data'))
return redirects
def task_owned_cache_paths(root):
# Same mapping used to construct the actual subprocess environments.
native = root / 'native'
return sorted([*native_redirects(native / 'preparation-support', preparation=True).values(),
*[path for index in range(3) for path in native_redirects(native / f'launch-{index}').values()]])
def storage_policy_check(args, b):
if args.storage_policy is None or args.trusted_storage_policy_sha256 is None:
raise RuntimeError('storage_bound_policy_unavailable; no launch permitted')
extended = resource_profile(getattr(args, 'protocol', PROTOCOLS[0])) == b.EXTENDED_PROFILE
policy = json.loads(read_trusted(args.storage_policy, args.trusted_storage_policy_sha256))
read_trusted(ROOT / STORAGE_APPROVAL, STORAGE_APPROVAL_SHA256)
fields = {'format', 'artifact_root', 'limit_bytes', 'scope', 'approval_sha256',
'native_paths', 'check_interval_seconds', 'max_native_growth_bytes', 'min_free_bytes',
'coverage_gaps', 'native_hard_cap', 'boot_id', 'mount_identity', 'not_after_boottime',
'shared_native_paths', 'task_owned_cache_paths', 'free_space_paths', 'free_space_mount_identities'}
root = args.artifact_root
protect_artifact_path(root)
if extended:
fields |= {'resource_profile', 'issued_boottime'}
if (root not in (b.EXTENDED_ROOT, b.SECOND_EXTENDED_ROOT) or policy.get('resource_profile') != b.EXTENDED_PROFILE
or type(policy.get('issued_boottime')) not in (int, float)
or not 0 <= policy['issued_boottime'] <= b.boottime()
or type(policy.get('not_after_boottime')) not in (int, float)
or not policy['issued_boottime'] < policy['not_after_boottime']
<= policy['issued_boottime'] + b.EXTENDED_TOTAL_SECONDS):
raise RuntimeError('storage_extended_profile_lifetime_mismatch')
if (set(policy) != fields or policy['format'] != (
b.EXTENDED_POLICY_FORMAT if extended else 'skillfactory.output-storage-policy.v4')
or not root.is_dir() or policy['artifact_root'] != str(root)
or type(policy['limit_bytes']) is not int
or not b.HEADROOM_BYTES + b.STDOUT_ALLOWANCE < policy['limit_bytes'] <= b.CAP_BYTES
or policy['scope'] != b.STORAGE_SCOPE or policy['approval_sha256'] != STORAGE_APPROVAL_SHA256
or policy['native_paths'] != native_paths(root) or policy['native_hard_cap'] is not False
or policy['coverage_gaps'] != list(NATIVE_GAPS)
or policy['shared_native_paths'] != shared_native_paths()
or policy['task_owned_cache_paths'] != task_owned_cache_paths(root)
or policy['free_space_paths'] != free_space_paths(root)
or policy['free_space_mount_identities'] != {p: mount_identity(Path(p)) for p in free_space_paths(root)}
or any(Path(p).is_relative_to(root) for p in shared_native_paths())
or type(policy['check_interval_seconds']) not in (int, float)
or not 0 < policy['check_interval_seconds'] <= 1
or type(policy['max_native_growth_bytes']) is not int
or not 0 < policy['max_native_growth_bytes'] <= b.CAP_BYTES
or type(policy['min_free_bytes']) is not int or policy['min_free_bytes'] < 2 * b.CAP_BYTES
or policy['boot_id'] != b.boot_id() or policy['mount_identity'] != mount_identity(root)
or type(policy['not_after_boottime']) not in (int, float)
or not b.boottime() < policy['not_after_boottime'] <= b.boottime() + (14400 if extended else 3600)):
raise RuntimeError('storage_policy_identity_or_conditions_mismatch')
return policy
class NativeScanError(RuntimeError):
"""Retain partial scan facts even when baseline fails before monitor creation."""
def __init__(self, report):
self.report = report
super().__init__('storage_native_scan_failed: ' + json.dumps(report, sort_keys=True))
def native_metadata(info):
return {key: getattr(info, 'st_' + key) for key in ('uid', 'gid', 'mode', 'dev', 'ino', 'ctime_ns')}
def native_directory_identity(info):
# Creating entries changes ctime during normal writes; it is not a new directory.
return (info.st_dev, info.st_ino, info.st_uid, info.st_gid, info.st_mode)
def native_directory_names(fd, *, recursive):
if not recursive:
return []
with os.scandir(fd) as entries:
return [entry.name for entry in entries]
def native_size(path, *, shared=False, strict_paths=(), exclude_paths=(), required=False, recursive=True):
"""Descriptor-relative traversal; never follow even a raced ancestor symlink.
Only an actual denied directory operation plus stable non-owned metadata and
an effective-credential access denial permits a shared coverage gap. A gap's
contents are not inspected. recursive=False observes only root/ancestor
filesystem facts, never directory entries. Missing ordinary entries are sampled churn, not
permission exclusions; disappearance during denial qualification is fatal.
"""
path = Path(path)
strict = tuple(Path(p) for p in strict_paths)
excluded = set(map(Path, exclude_paths))
result = {'logical_bytes': 0, 'permission_gaps': [], 'skipped_count': 0,
'disappeared_paths': [], 'symlink_paths': [], 'free_space': None}
def fail(current, operation, error, info=None):
result['failure'] = {'path': str(current), 'operation': operation,
'errno': error.errno, 'error': str(error),
'metadata': native_metadata(info) if info is not None else None}
raise NativeScanError(result) from error
def denied(parent, name, current, info, operation, error):
# An inaccessible ancestor of a strict path would also hide owned writes.
protected = any(current.is_relative_to(p) or p.is_relative_to(current) for p in strict)
if (not shared or protected or error.errno not in (errno.EACCES, errno.EPERM)
or info.st_uid in (os.getuid(), os.geteuid()) or not stat.S_ISDIR(info.st_mode)):
fail(current, operation, error, info)
try:
after = os.stat(name, dir_fd=parent, follow_symlinks=False)
accessible = os.access(name, os.R_OK | os.X_OK, dir_fd=parent,
effective_ids=True, follow_symlinks=False)
final = os.stat(name, dir_fd=parent, follow_symlinks=False)
except OSError as metadata_error:
result['denied_operation'] = {'path': str(current), 'errno': error.errno,
'metadata': native_metadata(info)}
fail(current, 'permission_metadata_recheck', metadata_error, info)
if (native_metadata(info) != native_metadata(after)
or native_metadata(info) != native_metadata(final) or accessible):
fail(current, 'permission_identity_or_access_changed', error, final)
result['permission_gaps'].append({'path': str(current), 'errno': error.errno,
'operation': operation, **native_metadata(final)})
result['skipped_count'] += 1
def observe_free(fd, current):
try:
usage = os.fstatvfs(fd)
result['free_space'] = {'observed_path': str(current),
'free_bytes': usage.f_bavail * usage.f_frsize,
'device': os.fstat(fd).st_dev, 'mount_identity': mount_identity(current)}
except OSError as error:
fail(current, 'free_space', error)
def visit(parent, name, current, *, ancestor=False):
if current in excluded:
return
try:
info = os.stat(name, dir_fd=parent, follow_symlinks=False)
except FileNotFoundError as error:
if required and (ancestor or current == path):
fail(current, 'required_directory', error)
result['disappeared_paths'].append(str(current))
return
except OSError as error:
fail(current, 'lstat', error)
if stat.S_ISLNK(info.st_mode):
# Explicit monitored roots/ancestors and owned trees cannot be aliases.
if ancestor or current == path or any(current.is_relative_to(p) or p.is_relative_to(current)
for p in strict):
fail(current, 'symlink', OSError(errno.ELOOP, 'monitored path is a symlink'), info)
result['symlink_paths'].append(str(current))
return
if ((ancestor or current == path or any(p.is_relative_to(current) for p in strict))
and not stat.S_ISDIR(info.st_mode)):
fail(current, 'directory', OSError(errno.ENOTDIR, 'not a directory'), info)
if stat.S_ISREG(info.st_mode):
if not os.access(name, os.R_OK, dir_fd=parent, effective_ids=True, follow_symlinks=False):
# access() returns False for ENOENT as well as permission denial.
# Atomic .pending publication may race the earlier lstat. Only a
# descriptor-relative ENOENT recheck is ordinary sampled churn;
# retain its last observed bytes conservatively, never a gap.
try:
os.stat(name, dir_fd=parent, follow_symlinks=False)
except FileNotFoundError:
result['disappeared_paths'].append(str(current))
result['logical_bytes'] += info.st_size
return
except OSError as error:
fail(current, 'file_access_recheck', error, info)
fail(current, 'file_access', PermissionError(None, 'file unreadable; access() exposes no errno'), info)
result['logical_bytes'] += info.st_size
return
if not stat.S_ISDIR(info.st_mode):
return
fd = None
operation = 'open'
try:
try:
fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent)
operation = 'fstat'
if native_directory_identity(os.fstat(fd)) != native_directory_identity(info):
fail(current, 'open_identity_changed', OSError(errno.ESTALE, 'directory changed'), info)
if ancestor:
walk_to_root(fd, current)
else:
if current == path:
observe_free(fd, current)
# Scandir errors belong to this directory, not one of its children.
operation = 'scandir'
names = native_directory_names(fd, recursive=recursive)
operation = 'children'
for child in names:
visit(fd, child, current / child)
operation = 'traversal_lstat'
after = os.stat(name, dir_fd=parent, follow_symlinks=False)
if native_directory_identity(after) != native_directory_identity(info):
fail(current, 'traversal_identity_changed', OSError(errno.ESTALE, 'directory changed'), after)
except OSError as error:
if operation in ('open', 'scandir') and error.errno in (errno.EACCES, errno.EPERM) and not ancestor:
denied(parent, name, current, info, operation, error)
else:
fail(current, operation, error, info)
finally:
if fd is not None:
os.close(fd)
def walk_to_root(fd, current):
observe_free(fd, current)
relative = path.relative_to(current)
if not relative.parts:
raise ValueError('native scan root must not be /')
child = current / relative.parts[0]
visit(fd, relative.parts[0], child, ancestor=child != path)
try:
with ExitStack() as stack:
fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
stack.callback(os.close, fd)
walk_to_root(fd, Path('/'))
except OSError as error:
fail(path, 'ancestor', error)
return result
class NativeMonitor:
def __init__(self, policy, b):
self.policy, self.b = policy, b
self.baseline = self.latest = {}
self.baseline_scans = self.latest_scans = {}
self.baseline_free_space = self.latest_free_space = {}
self.gap_history = []
self.growth = 0
self.next_check = 0
self.latest_scan_completed = False
try:
self.check(force=True)
except Exception as error:
if not isinstance(error, NativeScanError):
error.report = self.report()
raise
def check(self, *, force=False):
now = self.b.boottime()
if not force and now < self.next_check:
return
self.next_check = now + self.policy['check_interval_seconds']
self.latest_scan_completed = False
root = Path(self.policy['artifact_root'])
if mount_identity(root) != self.policy['mount_identity']:
raise RuntimeError('storage_policy_mount_identity_changed')
paths = self.policy['native_paths']
strict = [str(root), *self.policy['task_owned_cache_paths'],
*[p for p in paths if p not in self.policy['shared_native_paths']]]
scans = {}
self.latest_free_space = {}
checking_free_space = True
try:
for path in self.policy['free_space_paths']:
if mount_identity(Path(path)) != self.policy['free_space_mount_identities'][path]:
raise RuntimeError('storage_policy_free_space_mount_identity_changed: ' + path)
observation = native_size(path, required=True, recursive=False)
self.latest_free_space[path] = observation['free_space']
if observation['free_space']['mount_identity'] != self.policy['free_space_mount_identities'][path]:
raise RuntimeError('storage_policy_free_space_mount_identity_changed: ' + path)
if observation['free_space']['free_bytes'] < self.policy['min_free_bytes']:
raise RuntimeError('storage_native_free_space_stop')
checking_free_space = False
# Disjoint scans: output accessibility is NOT native growth/cap accounting.
# The unchanged pre-write aggregate cap still excludes only root/native.
for path in [str(root), *paths]:
exclusions = [p for p in [str(root), *paths] if p != path and Path(p).is_relative_to(path)]
scans[path] = native_size(path, shared=path in self.policy['shared_native_paths'],
strict_paths=strict, exclude_paths=exclusions, required=path == str(root))
except NativeScanError as error:
if checking_free_space:
self.latest_free_space[path] = error.report['free_space']
else:
scans[path] = error.report
self.latest_scans = scans
self.latest = {p: scans[p]['logical_bytes'] for p in paths if p in scans}
error.report = {**self.report(), 'failed_scan': error.report}
raise
previous_gaps = {gap['path']: gap for scan in self.latest_scans.values()
for gap in scan['permission_gaps']}
current_gaps = {gap['path']: gap for scan in scans.values() for gap in scan['permission_gaps']}
self.gap_history.append({'boottime': now, 'new': [current_gaps[p] for p in sorted(current_gaps.keys()
- previous_gaps.keys())],
'no_longer_observed': [previous_gaps[p] for p in sorted(previous_gaps.keys()
- current_gaps.keys())],
'changed': [{'before': previous_gaps[p], 'after': current_gaps[p]}
for p in sorted(previous_gaps.keys() & current_gaps.keys())
if previous_gaps[p] != current_gaps[p]]})
self.latest_scan_completed = True
self.latest_scans = scans
self.latest = {p: scans[p]['logical_bytes'] for p in paths}
if not self.baseline_scans:
self.baseline_free_space = dict(self.latest_free_space)
self.baseline_scans = scans
self.baseline = dict(self.latest)
self.growth = sum(max(0, size - self.baseline[p]) for p, size in self.latest.items())
if self.gap_history[-1]['changed']:
raise RuntimeError('storage_native_permission_gap_identity_changed')
if any(scan['free_space']['free_bytes'] < self.policy['min_free_bytes'] for scan in scans.values()):
raise RuntimeError('storage_native_free_space_stop')
if self.growth >= self.policy['max_native_growth_bytes']:
raise RuntimeError('storage_native_growth_stop')
def report(self):
return {'scope': self.b.STORAGE_SCOPE, 'native_hard_cap': False,
'coverage_complete': False, 'runtime_uid': os.geteuid(),
'latest_scan_completed': self.latest_scan_completed,
'baseline_skipped_count': sum(s['skipped_count'] for s in self.baseline_scans.values()),
'latest_skipped_count': sum(s['skipped_count'] for s in self.latest_scans.values()),
'baseline_logical_bytes': self.baseline, 'latest_logical_bytes': self.latest,
'observed_positive_growth_bytes': self.growth if self.latest_scan_completed else None,
'baseline_scans': self.baseline_scans, 'latest_scans': self.latest_scans,
'baseline_free_space': self.baseline_free_space, 'latest_free_space': self.latest_free_space,
'permission_gap_history': list(self.gap_history), 'policy': self.policy}
class ForwardingEvidence:
"""Stable runtime callbacks, installed before constructor; targets fail closed."""
recorder = None
observer = None
def __call__(self, event, state):
if self.recorder is None:
raise RuntimeError('recording_forwarder_uninstalled')
return self.recorder(event, state)
def seam(self, event, state):
if self.recorder is None:
raise RuntimeError('recording_forwarder_uninstalled')
return self.recorder.seam_callback(event, state)
def supplemental(self, state):
if self.observer is None:
raise RuntimeError('recording_observer_uninstalled')
return self.observer.supplemental(state)
def bounded_traceback(error):
"""Retain the innermost 32 frames, without locals, source lines or values."""
frames = []
omitted = 0
tb = error.__traceback__
while tb is not None:
code = tb.tb_frame.f_code
frames.append({'file': code.co_filename[-512:], 'line': tb.tb_lineno,
'function': code.co_name[:128]})
if len(frames) > 32:
del frames[0]
omitted += 1
tb = tb.tb_next
return {'frames': frames, 'omitted_outer_frames': omitted}
def external_failure(phase, error):
prefix = {'inference': 'inference', 'identity': 'identity', 'recording': 'recording',
'finalize': 'recording_persistence'}.get(phase, 'infrastructure')
if isinstance(error, (KeyboardInterrupt, InterruptedError)):
prefix = 'process_signal'
context = {'phase': phase, 'type': type(error).__name__, 'message': str(error),
'traceback': bounded_traceback(error)}
diagnostic = getattr(error, 'binding_diagnostic', None)
if diagnostic is not None:
context['binding_diagnostic'] = diagnostic
return {'causes': [prefix + '_external_failure'], 'fatal': True, 'context': context}
def execute_lifecycle(api, cfg, protocol, indices, output, budget, verify):
"""One constructor, stable forwards, actual step results, durable P2 transitions.
api is an explicit CPU-test seam; CLI always obtains it from real_runtime.
There is no plugin/provider flag or production verifier injection path.
"""
rows, failures = [], []
continuation_allowed = True
env = hooks = recorder = None
forward = ForwardingEvidence()
current = indices[0]
phase = 'identity'
counts = None
try:
verify(current)
budget.check()
with api.hooks(before_step=budget.before_step) as hooks:
try:
for position, current in enumerate(indices):
cfg.seed = protocol.trial(current).seed # sole declared transition
phase = 'identity'
verify(current) # same used cfg, never normalize here
budget.check()
phase = 'recording'
recorder = api.recorder(protocol, current, output / f'trial-{current:02d}',
budget_check=budget.recording, supplemental=forward.supplemental)
forward.recorder = recorder
stop = failure = None
acknowledged = False
report = None
try:
phase = 'initialization'
if position == 0:
env = api.environment(cfg=cfg, protocol=protocol, trial_index=current,
runtime_hooks=hooks, evidence_callback=forward,
seam_callback=forward.seam)
phase = 'identity'
verify(current) # immediate constructor boundary, before reset/action
phase = 'initialization'
# Enter only after construction, before any published reset snapshot.
observers = ExitStack()
try:
forward.observer = observers.enter_context(api.instrumentation(
env, backend_rng=api.backend_rng, generators={}))
except BaseException:
observers.close()
raise
observation, _ = env.reset()
else:
observation, _ = env.begin_trial(protocol, current)
stop = env.stop_result
if protocol.document['stage'] == 'P2' and current % 2:
phase = 'identity'
initial = recorder.audit.report().get('initial_pair_state')
if (not rows or rows[-1]['trial']['index'] != current - 1
or not isinstance(initial, dict) or not initial
or rows[-1].get('initial_pair_state') != initial):
raise RuntimeError('identity_pair_initial_state_mismatch')
tick = 0
while stop is None:
budget.check()
phase = 'inference'
x = observation['policy'] # raw 46D evidence, not normalized model features
action, mean, std = api.infer(x, protocol.trial(current).controller_kind)
phase = 'recording'
causes = recorder.before_control(tick, x, action, mean=mean, std=std)
if causes:
raise RuntimeError('Rejected control evidence: ' + repr(causes))
phase = 'step'
actual = env.step(action)
stop = actual.stop
tick += actual.ticks_advanced
phase = 'recording'
causes = recorder.after_step(actual)
if causes:
raise RuntimeError('Rejected actual step evidence: ' + repr(causes))
observation = actual.observation
except BaseException as error:
failure = external_failure(phase, error)
failures.append(failure)
if env is not None:
stop = env.stop_result # never manufacture StopResult/actions/ticks
finally:
def acknowledge(exact):
nonlocal acknowledged
env.acknowledge_finalized(exact)
acknowledged = True
try:
phase = 'finalize'
report = recorder.finalize(stop=stop, external_failure=failure,
acknowledge=acknowledge if failure is None else None)
except BaseException as error:
failures.append(external_failure('finalize', error))
report = recorder.audit.report()
report.update(status='persistence_failed', functional_passed=False)
# Preserve partial stream; close it without claiming durability.
try:
recorder.stream.close()
except BaseException as close_error:
failures.append(external_failure('finalize', close_error))
rows.append(report)
if failure is not None or failures or stop is None or stop.fatal or not acknowledged:
continuation_allowed = False
break
recorder = None
finally:
phase = 'cleanup'
try:
if 'observers' in locals():
observers.close()
finally:
if env is not None:
env.close() # hooks still installed, cleanup steps cannot evade counting
except BaseException as error:
failures.append(external_failure(phase, error))
finally:
if hooks is not None:
counts = {name: getattr(hooks, name) for name in (
'attempted_steps', 'completed_steps', 'initialization_steps', 'rollout_steps')}
try:
verify(current) # used actual cfg, after close/guard unwind, including construction failure
except BaseException as error:
failures.append(external_failure('identity', error))
matrix = api.matrix(protocol, rows)
matrix.update(failures=failures, observed_counts=counts, selected_rows=indices, last_trial_index=current,
equality_scope='recorded_application_controller_and_enumerated_observable_rng_only',
complete_backend_state_equality=False, deterministic_replay=False,
continuation_eligible=continuation_allowed and not failures and all(
row.get('terminal', {}).get('runtime_stop') is not None
and not row['terminal']['runtime_stop']['fatal'] for row in rows)
and len(rows) == len(indices), accepted_skill=False, optimizer_steps=0)
return matrix
def preimport_replicator():
"""Only the intended optional package may be absent; broken dependencies fail."""
missing = None
try:
importlib.import_module(OPTIONAL_REPLICATOR)
except ModuleNotFoundError as error:
if error.name not in ('omni.replicator', OPTIONAL_REPLICATOR):
raise
missing = error.name
return {'module': OPTIONAL_REPLICATOR, 'available': missing is None,
'missing_module': missing, 'sampling_scope': 'no configured sampling',
'scope_requires_nominal_config_qualification': True,
'complete_rng_state_claim': False}
def check_replicator_inventory(inventory):
"""Availability is bound by exact reviewed module membership, not a grant."""
if (OPTIONAL_REPLICATOR in sys.modules) != (OPTIONAL_REPLICATOR in inventory['modules']):
raise RuntimeError('identity_optional_replicator_availability_changed')
def stored_config_fields(value):
"""Binding's dataclass/ClassVar/instance/slot field set, without properties."""
missing = object()
def stored(name):
item = inspect.getattr_static(value, name, missing)
if item is missing:
raise ValueError('Unstored dataclass config field: ' + name)
if type(item) is MemberDescriptorType:
return item.__get__(value, type(value))
if inspect.getattr_static(type(item), '__get__', None) is not None:
# Functions/classes are legitimate stored values, not evaluated descriptors.
if not (inspect.isfunction(item) or inspect.isclass(item) or inspect.isbuiltin(item)):
raise ValueError('Unsupported config field descriptor: ' + name)
return item
fields = {name: stored(name) for name in inspect.getattr_static(type(value), '__dataclass_fields__')}
dictionary = next((vars(cls)['__dict__'] for cls in type(value).__mro__ if '__dict__' in vars(cls)), None)
if type(dictionary) is GetSetDescriptorType:
fields.update(dictionary.__get__(value, type(value)))
elif dictionary is not None:
raise ValueError('Unsupported config instance dictionary descriptor')
for cls in type(value).__mro__:
slots = vars(cls).get('__slots__', ())
for slot in (slots,) if type(slots) is str else slots:
if slot not in ('__dict__', '__weakref__') and inspect.getattr_static(value, slot, missing) is not missing:
with suppress(AttributeError): # uninitialized slot, as in binding._config_fields
fields[slot] = stored(slot)
return fields
def config_references(config, binding):
"""Collect exact ResolvableStrings, rejecting invalid graphs before imports."""
references, active = [], set()
def visit(value):
if binding._resolvable(value):
reference = str(value)
if reference.count(':') != 1:
raise ValueError('Unsupported ResolvableString reference')
module, attribute = reference.split(':')
if not attribute.isidentifier() or not all(p.isidentifier() for p in module.split('.')):
raise ValueError('Unsupported ResolvableString reference')
references.append(value)
return
if value is None or type(value) in (str, bool, int, Path, PosixPath, WindowsPath):
return
if type(value) is float:
if not math.isfinite(value):
raise ValueError('Nonfinite effective config')
return
if inspect.isclass(value) or inspect.isfunction(value) or inspect.isbuiltin(value):
return
if id(value) in active:
raise ValueError('Cyclic effective config')
active.add(id(value))
try:
if isinstance(value, enum.Enum):
children = (value.value,)
elif type(value) in (dict, MappingProxyType):
if any(type(k) is not str for k in value):
raise ValueError('Effective config mapping keys must be strings')
children = value.values()
elif type(value) in (tuple, list):
children = value
elif type(value) is slice:
children = (value.start, value.stop, value.step)
elif type(value) is functools.partial:
children = (value.func, value.args, value.keywords)
elif dataclasses.is_dataclass(type(value)):
children = stored_config_fields(value).values()
else:
raise ValueError('Unsupported effective config type: ' + type(value).__qualname__)
for child in children:
visit(child)
finally:
active.remove(id(value))
visit(config)
return references
def config_module_path(name, binding):
"""Only exact file-backed references in the existing bound source namespaces.
No find_spec parent imports, prefix wildcard imports, or external inventory
inference. Loaded parent search paths must also be the installed directories.
"""
roots = {'isaaclab': binding.ISAAC_ROOT / 'source/isaaclab/isaaclab',
'isaaclab_physx': binding.ISAAC_ROOT / 'source/isaaclab_physx/isaaclab_physx',
'SkillFactory': ROOT / binding.PACKAGE}
parts = name.split('.')
if parts[0] not in roots:
raise ValueError('Config reference requires explicit external inventory: ' + name)
directory = roots[parts[0]]
for index in range(len(parts)):
if index:
directory = directory / parts[index]
package = directory / '__init__.py'
path = package if package.is_file() else directory.with_suffix('.py')
if not path.is_file():
raise ModuleNotFoundError('Missing installed config module: ' + name, name=name)
path = binding._file(path)
parent_name = '.'.join(parts[:index + 1])
module = sys.modules.get(parent_name)
if module is not None:
if binding._module_file(parent_name) != path:
raise ValueError('Config reference installed source mismatch: ' + parent_name)
if path == package and list(vars(module).get('__path__', ())) != [str(directory)]:
raise ValueError('Config reference package search path mismatch: ' + parent_name)
if index == 0 and module is None:
raise ValueError('Config reference requires already imported installed root: ' + parent_name)
if index < len(parts) - 1 and path != package:
raise ValueError('Config reference parent is not an installed package: ' + parent_name)
return path
def preimport_config_references(config, binding):
"""Explicit runner import phase; never calls references or changes their caches.
The unchanged encoder/import inventory binds these modules and their source /
direct dependency closure later. This report is discovery, not a trust grant.
"""
references = config_references(config, binding)
names = sorted({str(value).split(':')[0] for value in references})
paths = {name: config_module_path(name, binding) for name in names}
report = {'references': sorted({str(value) for value in references}), 'modules': {}}
for name, expected in paths.items():
importlib.import_module(name) # absence and transitive failures deliberately propagate
path = binding._module_file(name)
if path != expected or config_module_path(name, binding) != expected:
raise ValueError('Config reference imported source mismatch: ' + name)
report['modules'][name] = {'path': str(path), 'sha256': binding.sha256(path.read_bytes())}
for value in references:
binding._reference_target(value, set()) # static named-symbol/cache validation only
return report
def preforward_video_recorder(cfg, env_module):
"""Mirror pinned ManagerBasedRLEnv.__init__ forwarding before config binding."""
env_module.verify_installed_stepping(Path('/home/cen/IsaacLab/source/isaaclab/isaaclab/envs'))
# EvaluationContactCycleEnv calls super().__init__(cfg), i.e. render_mode=None.
if cfg.video_recorder is not None:
cfg.video_recorder.env_render_mode = None
cfg.video_recorder.eye = tuple(float(x) for x in cfg.viewer.eye)
cfg.video_recorder.lookat = tuple(float(x) for x in cfg.viewer.lookat)
def preimport_constructor_modules(binding):
"""Eagerly resolve the finite reviewed constructor set; never hide new imports.
Transitive imports remain subject to the unchanged full binding inventory and
immediate postconstructor/final equality checks, not an import ignorelist.
"""
report = {}
for name, (filename, expected_hash) in CONSTRUCTOR_IMPORT_PINS.items():
expected = Path(filename)
if (config_module_path(name, binding) != expected
or binding.sha256(expected.read_bytes()) != expected_hash):
raise ValueError('Constructor import source mismatch: ' + name)
importlib.import_module(name)
if binding._module_file(name) != expected:
raise ValueError('Constructor imported source mismatch: ' + name)
report[name] = {'path': filename, 'sha256': binding.sha256(expected.read_bytes())}
if report[name]['sha256'] != expected_hash:
raise ValueError('Constructor import source changed: ' + name)
return report
# Same-named eager submodule imports bypass this package's lazy function export.
PHYSX_EXPORT_SOURCES = {
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/__init__.pyi':
'811bf1b50a42f0c854f13a8c26a9cf1a00e00120132c633c38b58375908f8dc6',
'/home/cen/isaacsim-6.0.1/kit/python/lib/python3.12/site-packages/lazy_loader/__init__.py':
'9017e7e780ffe7742b8cd25882c22c93acb9ac353f3372b8dcc76acf832acf29',
}
def physx_constructor_export(binding, *, resolve=False, expected=None):
"""Resolve once before binding; later checks never repair namespace drift."""
package_name = 'isaaclab_physx.cloner'
module_name = package_name + '.physx_replicate'
sources = dict(PHYSX_EXPORT_SOURCES)
for name in (package_name, module_name):
filename, digest = CONSTRUCTOR_IMPORT_PINS[name]
if binding._module_file(name) != Path(filename):
raise ValueError('identity_physx_export_module_path: ' + name)
sources[filename] = digest
for filename, digest in sources.items():
if binding.sha256(Path(filename).read_bytes()) != digest:
raise ValueError('identity_physx_export_source: ' + filename)
package, module = sys.modules[package_name], sys.modules[module_name]
target = vars(module).get('physx_replicate')
if (not inspect.isfunction(target) or target.__module__ != module_name
or target.__qualname__ != 'physx_replicate'):
raise ValueError('identity_physx_export_function')
binding._source_code(target, Path(CONSTRUCTOR_IMPORT_PINS[module_name][0]))
if expected is not None and target is not expected:
raise ValueError('identity_physx_export_changed_callable')
if resolve:
resolver = vars(package).get('__getattr__')
loader_path = Path(next(p for p in PHYSX_EXPORT_SOURCES if '/lazy_loader/' in p))
if not inspect.isfunction(resolver) or resolver.__module__ != 'lazy_loader':
raise ValueError('identity_physx_export_resolver')
binding._source_code(resolver, loader_path)
if resolver('physx_replicate') is not target:
raise ValueError('identity_physx_export_resolution')
if vars(package).get('physx_replicate') is not target:
raise ValueError('identity_physx_export_namespace')
return target, {'package': package_name, 'symbol': 'physx_replicate',
'module': module_name, 'qualname': target.__qualname__, 'sources': sources}
def verify_constructor_export(runtime):
_, metadata = physx_constructor_export(runtime.binding, expected=runtime.constructor_export)
if metadata != runtime.generator_inventory['constructor_export']:
raise ValueError('identity_physx_export_metadata')
def real_runtime(args, output):
"""Actual imported runtime; no environment/sim.step here, including preparation."""
os.chdir(ROOT)
from isaaclab.app import AppLauncher
app_module = sys.modules[AppLauncher.__module__]
if not str(Path(app_module.__file__).resolve()).startswith('/home/cen/IsaacLab/source/'):
raise RuntimeError('identity_require_actual_/home/cen/IsaacLab_imports')
app = AppLauncher(headless=True, device=args.device, fast_shutdown=True,
kit_args=f'--/log/file={Path(args.native_output) / "kit.log"}').app
try:
inventory = (None if getattr(args, 'discover_inventory', False)
else json.loads(Path(args.inventory).read_bytes()))
# Execution always uses reviewed eager imports; discovery is preparation-only.
for name in (() if inventory is None else inventory['modules']):
importlib.import_module(name)
modules = {name: importlib.import_module(TASK + name) for name in (
'evaluation_cfg', 'evaluation_env', 'evaluation_recording',
'evaluation_instrumentation', 'evaluation_rng_scope')}
for name in ('evaluation_protocol', 'evaluation_audit', 'evaluation_binding',
'evaluation_budget', 'evaluation_adoption', 'replay'):
modules[name] = importlib.import_module(LEG + name)
ins = modules['evaluation_instrumentation']
for name in ins.BACKEND_SOURCE_INVENTORY:
importlib.import_module(name)
# ManagerBasedEnv.seed treats Replicator as optional. Do not enable an
# extension, mask transitive defects, or claim its hidden RNG is observed.
replicator = preimport_replicator()
print(json.dumps({'optional_replicator': replicator}), flush=True)
if inventory is not None:
check_replicator_inventory(inventory)
# Numerical lazy paths also belong in the reviewed inventory.
import pytest
import torch
if torch.device(args.device).type == 'cuda':
torch.cuda.init()
torch.cuda.get_rng_state(torch.device(args.device))
ins.verify_backend_sources()
replay = modules['replay']
if not (ROOT / replay.CANDIDATE).is_file():
raise RuntimeError('identity_real_artifact_test_cannot_skip_missing_candidate')
# Required unchanged CPU test in THIS file-backed, correctly imported process.
if pytest.main(['-q', '-p', 'no:cacheprovider', REAL_TEST]) != 0:
raise RuntimeError('identity_real_artifact_CPU_test_failed')
cfg = modules['evaluation_cfg'].EvaluationContactCycleEnvCfg()
protocol = modules['evaluation_protocol'].load_protocol(ROOT / args.protocol)
cfg.sim.device = args.device
cfg.seed = protocol.trial(args.trial_index).seed
binding = modules['evaluation_binding']
for name in binding.REQUIRED_RUNTIME:
importlib.import_module(name)
preforward_video_recorder(cfg, modules['evaluation_env'])
constructor_imports = preimport_constructor_modules(binding)
constructor_export, export_metadata = physx_constructor_export(binding, resolve=True)
config_imports = preimport_config_references(cfg, binding)
print(json.dumps({'config_reference_imports': config_imports}), flush=True)
binding.normalize_config(cfg) # exactly once, actual config used later
scope = modules['evaluation_rng_scope']
inventory_report = scope.qualify_config(cfg)
inventory_report['optional_replicator'] = replicator
inventory_report['config_reference_imports'] = config_imports
inventory_report['constructor_imports'] = constructor_imports
inventory_report['constructor_export'] = export_metadata
physx_constructor_export(binding, expected=constructor_export)
ins.qualify_backend_rng(scope.backend_rng(None))
arguments = dict(root=ROOT, protocol_path=args.protocol, effective_config=cfg,
future_sources=FUTURE_SOURCES, declared_overrides={'seed': 'protocol.trial.seed'},
dependency_inventory=inventory, entrypoints=(SCRIPT,))
api = SimpleNamespace(
environment=modules['evaluation_env'].EvaluationContactCycleEnv,
hooks=modules['evaluation_env'].RuntimeHooks.isaac,
recorder=modules['evaluation_recording'].EvaluationRecorder,
instrumentation=ins.EvaluationInstrumentation, backend_rng=scope.backend_rng,
matrix=modules['evaluation_audit'].matrix_report,
)
return SimpleNamespace(app=app, cfg=cfg, protocol=protocol, binding=binding,
arguments=arguments, api=api, replay=replay, torch=torch,
generator_inventory=inventory_report, constructor_export=constructor_export)
except BaseException:
import traceback
traceback.print_exc()
app.close(exit_code=1) # supported fast shutdown must preserve failure
raise
def runtime_metadata(runtime):
torch = runtime.torch
device = torch.device(runtime.cfg.sim.device)
driver = Path('/proc/driver/nvidia/version')
return {'torch': str(torch.__version__), 'numpy': str(importlib.import_module('numpy').__version__),
'warp': str(importlib.import_module('warp').__version__), 'device': str(device),
'device_name': torch.cuda.get_device_name(device) if device.type == 'cuda' else 'CPU',
'torch_cuda': torch.version.cuda,
'driver': driver.read_text() if driver.exists() else 'unavailable',
'generator_inventory': runtime.generator_inventory,
'evidence_scope': importlib.import_module(TASK + 'evaluation_rng_scope').EVIDENCE_SCOPE,
'unobserved_internals': importlib.import_module(TASK + 'evaluation_rng_scope').UNOBSERVED_INTERNALS}
class InventoryPreparationError(ValueError):
"""Unreviewed diagnostic only; partial identities cannot become a manifest."""
def __init__(self, diagnostics):
self.report = {'status': 'FAIL', 'inventory_status': 'unreviewed_diagnostic',
'execution_authorized': False, 'environment_constructed': False,
'sim_step_calls': 0, 'invalid_entries': diagnostics,
'note': 'Invalid entries were not hashed or accepted; discovery is incomplete.'}
super().__init__('Inventory preparation FAIL: ' + '; '.join(
row['entry'] + ': ' + row['error'] for row in diagnostics))
def proposed_native_layouts(roots):
"""Independent known layouts only for roots with validated file identities."""
layouts = {}
if 'torch' in roots:
layouts['torch'] = (roots['torch'] / 'lib',)
if 'numpy' in roots:
layouts['numpy'] = (roots['numpy'] / ('_core' if (roots['numpy'] / '_core').is_dir() else 'core'),
roots['numpy'] / 'random', roots['numpy'] / 'linalg', roots['numpy'] / 'fft',
roots['numpy'].parent / 'numpy.libs')
if 'warp' in roots:
layouts['warp'] = (roots['warp'] / 'bin',)
return layouts
def proposed_inventory(binding, cfg):
"""Collect independent incompatibilities, never approve a partial inventory.
Inspect only valid imported identities. An invalid provider is not traversed;
other independent source/module/native entries can still be diagnosed.
"""
diagnostics = []
def invalid(stage, entry, error):
diagnostics.append({'stage': stage, 'entry': entry, 'error': str(error),
'module_observation': binding.module_diagnostic(entry)})
_, names = binding._sources(ROOT, FUTURE_SOURCES, (SCRIPT,))
roles = set(names)
binding._encode(cfg, names, set())
source_records = binding._imports(names, ROOT, diagnostics=diagnostics)
sources = set(source_records)
numerical = binding.NUMERICAL_ROOTS
required = binding.required_dependencies(sources, roles)
modules = {}
for name in sorted(required):
try:
# Required modules must already be imported by the actual runtime.
# Never interpret incidental __module__ metadata as an import name.
modules[name] = binding._dependency_module_identity(name)
except (ValueError, OSError, ImportError) as error:
invalid('dependency-module', name, error)
roots = {name: Path(modules[name]['path']).parent for name in numerical if name in modules}
layouts = proposed_native_layouts(roots)
artifacts = {}
def add(path, owner, vendor=False):
try:
path = binding._file(path)
label = str(path)
if label in artifacts:
raise ValueError('Duplicate proposed native artifact: ' + label)
artifacts[label] = {'module': owner, 'path': label, 'sha256': binding.sha256(path.read_bytes()),
'purpose': 'Installed numerical native implementation consumed by ' + owner}
if vendor:
artifacts[label]['vendor'] = 'NVIDIA'
except (ValueError, OSError) as error:
invalid('native-artifact', str(path), error)
for owner, directories in layouts.items():
for directory in directories:
paths = sorted(directory.glob('*.so*'))
if not paths:
invalid('native-layout', str(directory), 'Unsupported/missing numerical native layout')
for path in paths:
add(path, owner)
if 'torch' in roots and (roots['torch'] / 'lib/libtorch_cuda.so').is_file():
for package in binding.NVIDIA_CUDA_PACKAGES:
directory = roots['torch'].parent / 'nvidia' / package / 'lib'
paths = sorted(directory.glob('*.so*'))
if package in ('cuda_runtime', 'cublas', 'cudnn') and not paths:
invalid('native-layout', str(directory), 'Unsupported/missing NVIDIA CUDA native layout')
for path in paths:
add(path, 'torch', vendor=True)
if diagnostics:
raise InventoryPreparationError(diagnostics)
proposal = {'scope': binding.INVENTORY_SCOPE, 'modules': modules, 'artifacts': artifacts}
try:
return binding._dependency_inventory(proposal, sources, roles)
except (ValueError, OSError, ImportError) as error:
invalid('inventory-validation', 'proposal', error)
raise InventoryPreparationError(diagnostics) from error
def bound_filesystem_inputs(manifest):
"""Extract immutable file identities, not runtime objects, for external rehash."""
document = json.loads(manifest)
files, links = {}, {}
def visit(value):
if isinstance(value, dict):
if {'path', 'target', 'device', 'inode', 'target_sha256'} <= value.keys():
links[value['path']] = dict(value)
if isinstance(value.get('path'), str) and isinstance(value.get('sha256'), str):
path, digest = value['path'], value['sha256']
if path in files and files[path] != digest:
raise ValueError('identity_conflicting_file_hash: ' + path)
files[path] = digest
for item in value.values():
visit(item)
elif isinstance(value, list):
for item in value:
visit(item)
visit(document)
files.update(document.get('runtime_layout_pins', {}))
# Frozen bundle verification remains the unchanged child's verify_bundle.
# Independently inventory every immutable prepared01 file for postexit equality.
frozen = ROOT / 'logs/contact_cycle/sim_residual_training_v1/prepared01'
for path in frozen.rglob('*'):
if path.is_file():
files[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
return {'files': files, 'links': links, 'frozen_root': str(frozen),
'scope': 'immutable-files-only; no postexit same-object cfg claim'}
def verify_filesystem_inputs(evidence):
for filename, expected in evidence.get('links', {}).items():
path = Path(filename)
stat = path.lstat()
target = str(path.readlink())
if (target != expected['target'] or stat.st_dev != expected['device']
or stat.st_ino != expected['inode']
or hashlib.sha256(target.encode()).hexdigest() != expected['target_sha256']):
raise ValueError('identity_postexit_link_changed: ' + filename)
for filename, expected in evidence['files'].items():
path = Path(filename)
if (not path.is_absolute() or path != path.resolve(strict=True) or path.is_symlink()
or hashlib.sha256(path.read_bytes()).hexdigest() != expected):
raise ValueError('identity_postexit_file_changed: ' + filename)
frozen = Path(evidence['frozen_root'])
expected = {p for p in evidence['files'] if Path(p).is_relative_to(frozen)}
if expected != {str(p) for p in frozen.rglob('*') if p.is_file()}:
raise ValueError('identity_postexit_frozen_file_set_changed')
return {'status': 'PASS', 'files_rehashed': len(evidence['files']), 'scope': evidence['scope']}
def prepare(args):
discovery = getattr(args, 'discover_inventory', False)
if (args.output is None or args.artifact_root is None
or (args.inventory is None and not discovery) or (args.inventory is not None and discovery)):
raise ValueError('Preparation requires root/output and exactly one of reviewed inventory or discovery')
protect_artifact_path(args.output)
b = cpu_module('evaluation_budget')
protect_artifact_path(args.artifact_root)
if not args.output.is_relative_to(args.artifact_root) or args.output == args.artifact_root:
raise ValueError('Preparation outputs must belong to aggregate artifact root')
b.durable_directory(args.output)
native = args.artifact_root / 'native'
if native != native.resolve():
raise RuntimeError('storage_native_redirect_symlink')
if not native.exists():
b.durable_directory(native)
native = native / args.output.name
b.durable_directory(native)
args.native_output = str(native)
def write(path, data):
if (b.artifact_size(args.artifact_root, exclude_native=True) + len(data)
> b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE):
raise RuntimeError('storage_aggregate_cap')
b.durable_write(path, data)
runtime = None
completed = False
started = b.boottime()
try:
runtime = real_runtime(args, args.output)
if discovery:
try:
inventory = proposed_inventory(runtime.binding, runtime.cfg)
except InventoryPreparationError as error:
error.report['module_observations'] = {
name: runtime.binding.module_diagnostic(name) for name, module in tuple(sys.modules.items())
if isinstance(module, ModuleType)}
write(args.output / 'inventory.diagnostic.json', b.encode(error.report))
raise
write(args.output / 'inventory.proposed.json', b.encode(inventory))
runtime.arguments['dependency_inventory'] = inventory
manifest = runtime.binding.prepare_binding(trial_index=args.trial_index, **runtime.arguments)
write(args.output / 'binding.json', manifest)
write(args.output / 'incidental-observations.json', b.encode(
runtime.binding.incidental_observations(
set(json.loads(manifest)['imports']) | set(runtime.arguments['dependency_inventory']['modules']))))
metadata = runtime_metadata(runtime)
verify_constructor_export(runtime)
runtime.binding.verify_binding(manifest, trusted_manifest_sha256=hashlib.sha256(manifest).hexdigest(),
trial_index=args.trial_index, **runtime.arguments)
filesystem = bound_filesystem_inputs(manifest)
verify_filesystem_inputs(filesystem)
write(args.output / 'preclose-filesystem.json', b.encode(filesystem))
write(args.output / 'preparation.json', b.encode({
'storage_scope': b.STORAGE_SCOPE, 'native_hard_cap': False, 'launch_identity': launch_identity(),
'manifest_sha256_for_independent_review': hashlib.sha256(manifest).hexdigest(),
'execution_authorized': False, 'environment_constructed': False, 'sim_step_calls': 0,
'inventory_status': 'unreviewed_proposal' if discovery else 'caller_supplied',
'binding_format': runtime.binding.FORMAT, 'evidence_scope': runtime.binding.PROVENANCE_SCOPE,
'limitations': runtime.binding.PROVENANCE_LIMITATIONS, 'accepted_skill': False,
'prepare_seconds': b.boottime() - started, 'runtime': metadata,
'identity_order': ('actual cfg and inputs verified; evidence durable; then app.close; '
'parent postexit files/cleanup'),
'same_object_cfg_scope': 'preclose only; no claim after interpreter destruction',
'preclose_identity_verified': True, 'shutdown': 'supported-default-fast',
'final_pass_owner': 'external supervisor after exit',
'note': 'Preparation timing is not an actual backend rollout/storage qualification.'}))
completed = True
except BaseException as error:
write(args.output / 'preparation-failure.json', b.encode({
'status': 'FAIL', 'error': repr(error), 'failure': external_failure('preparation', error)}))
import traceback
traceback.print_exc()
raise
finally:
if runtime is not None:
runtime.app.close(exit_code=0 if completed else 1)
return 0
def ancestor(parent, child):
for _ in range(32):
fields = Path(f'/proc/{child}/stat').read_text().rsplit(') ', 1)[1].split()
child = int(fields[1])
if child == parent:
return True
if child <= 1:
return False
return False
def worker_payload(fd):
"""Single-use inherited socket; only a live matching supervisor may hand off."""
with socket.socket(fileno=fd) as channel:
channel.settimeout(15)
pid, uid, _ = struct.unpack('3i', channel.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
command = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0')
if (Path(f'/proc/{pid}/exe').resolve() != Path('/usr/bin/python3.12')
or not ancestor(pid, os.getpid())) or uid != os.getuid() or b'--run' not in command or not any(
item.decode(errors='replace').endswith(SCRIPT) for item in command):
raise RuntimeError('identity_inherited_parent_supervisor_required')
channel.sendall(str(os.getpid()).encode() + b'\n')
with channel.makefile('rb') as stream:
data = stream.read(1 << 20)
if len(data) >= 1 << 20:
raise RuntimeError('identity_oversized_handoff')
return json.loads(data)
def run_worker(payload):
b = cpu_module('evaluation_budget')
if payload.get('launch_identity') != launch_identity():
raise RuntimeError('identity_worker_launch_changed')
args = SimpleNamespace(**payload['arguments'])
for name in ('artifact_root', 'inventory', 'manifest'):
setattr(args, name, Path(getattr(args, name)))
reservation = payload['reservation']
index = reservation['index']
output = args.artifact_root / f'launch-{index}'
args.trial_index = reservation['grant']['trial_indices'][0]
budget = b.LaunchBudget(args.artifact_root, reservation)
ledger = json.loads((args.artifact_root / 'budget.json').read_bytes())
if ledger['launches'][-1] != reservation or reservation['status'] != 'reserved':
raise RuntimeError('identity_stale_reservation')
budget.write(output / 'worker-claim.json', {'pid': os.getpid(), 'reservation': reservation})
handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True))
for s in (signal.SIGTERM, signal.SIGINT)}
runtime = None
result = {'accepted_skill': False, 'optimizer_steps': 0, 'continuation_eligible': False,
'storage_scope': b.STORAGE_SCOPE,
'observed_counts': None, 'failures': [],
'rows': [{'trial_index': i, 'status': 'not_run'}
for i in range(20 if protocol_stage(args.protocol) == 'P2' else 2)]}
verify = None
try:
budget.check()
manifest = read_trusted(args.manifest, args.trusted_manifest_sha256)
if b.digest(args.inventory) != reservation['grant']['inventory_sha256']:
raise RuntimeError('identity_inventory_changed')
runtime = real_runtime(args, output)
def verify(row):
budget.check(cleanup=True)
verify_constructor_export(runtime)
check_replicator_inventory(runtime.arguments['dependency_inventory'])
return runtime.binding.verify_binding(
manifest, trusted_manifest_sha256=args.trusted_manifest_sha256,
trial_index=row, **runtime.arguments)
verify(args.trial_index)
frozen, _ = runtime.replay.verify_bundle(ROOT)
policy = runtime.replay.load_frozen_policy(ROOT / runtime.replay.CANDIDATE, frozen)
policy = policy.to(runtime.cfg.sim.device)
def infer(x, kind):
if kind not in ('learned_policy', 'analytic_reference'):
raise ValueError('identity_unknown_controller')
mode = 'learned' if kind == 'learned_policy' else 'reference'
return runtime.replay.inference_action(x, mode, policy if mode == 'learned' else None)
runtime.api.infer = infer
budget.write(output / 'runtime.json', runtime_metadata(runtime))
result = execute_lifecycle(runtime.api, runtime.cfg, runtime.protocol,
reservation['grant']['trial_indices'], output, budget, verify)
except BaseException as error:
result['failures'].append(external_failure('initialization', error))
result['continuation_eligible'] = False
finally:
if runtime is not None and verify is not None:
try:
verify(result.get('last_trial_index', args.trial_index))
filesystem = bound_filesystem_inputs(manifest)
verify_filesystem_inputs(filesystem)
budget.write(output / 'preclose-filesystem.json', filesystem)
result['preclose_identity_verified'] = True
except BaseException as error:
result['failures'].append(external_failure('identity', error))
result['continuation_eligible'] = False
result['same_object_cfg_scope'] = 'after env.close, before app.close only'
result['final_pass_owner'] = 'external supervisor after exit'
result['storage_scope'] = b.STORAGE_SCOPE
result['native_hard_cap'] = False
persisted = False
try:
budget.write(output / 'result.json', result)
persisted = True
finally:
for sig, handler in handlers.items():
signal.signal(sig, handler)
if runtime is not None:
runtime.app.close(exit_code=0 if persisted and result['continuation_eligible'] else 1)
return 0 if result['continuation_eligible'] else 1
def process_table():
table = {}
for path in Path('/proc').glob('[0-9]*/stat'):
try:
fields = path.read_text().rsplit(') ', 1)[1].split()
table[int(path.parent.name)] = (int(fields[19]), int(fields[1]), int(fields[2]), fields[0])
except (FileNotFoundError, ProcessLookupError):
pass
return table
class OwnedProcesses:
"""Dedicated sole-spawner supervisor only; not general service containment.
Pre-existing child trees are refused. Track birth identities, not names/PIDs;
orphan attribution relies on this process spawning no other jobs after start.
"""
def __init__(self):
table = process_table()
self.supervisor = (os.getpid(), table[os.getpid()][0])
if any(row[1] == os.getpid() for row in table.values()):
raise RuntimeError('process_dedicated_supervisor_has_preexisting_children')
self.preexisting = {(pid, row[0]) for pid, row in table.items()}
self.known = set()
self.root = None
self.unknown = False
def attach(self, child):
row = process_table().get(child.pid)
if row is None:
raise RuntimeError('process_spawn_identity_unavailable')
self.root = (child.pid, row[0])
self.known.add(self.root)
def discover(self):
table = process_table()
if table.get(self.supervisor[0], (None,))[0] != self.supervisor[1]:
self.unknown = True
return {}
# Retain exclusions for observed descendants of unrelated baseline
# processes as well, even if their parent later exits/adopts here.
changed = True
while changed:
changed = False
for pid, row in table.items():
identity = (pid, row[0])
parent = table.get(row[1])
if (identity not in self.preexisting and identity not in self.known and parent is not None
and (row[1], parent[0]) != self.supervisor
and (row[1], parent[0]) in self.preexisting):
self.preexisting.add(identity)
changed = True
if any(row[1] == self.supervisor[0] and (pid, row[0]) in self.preexisting
for pid, row in table.items()):
self.unknown = True # unexpected unrelated adoption violates dedicated ownership
active = {pid: row for pid, row in table.items() if (pid, row[0]) in self.known}
changed = True
while changed:
changed = False
for pid, row in table.items():
identity = (pid, row[0])
if pid in active or identity in self.preexisting:
continue
# Direct children after attach are adopted owned orphans under
# the enforced dedicated, sole-spawner architecture.
if (self.root is not None or self.unknown) and (row[1] == self.supervisor[0] or row[1] in active):
self.known.add(identity)
active[pid] = row
changed = True
return active
def signal(self, pid, row, sig):
# pidfd pins the process during signal delivery; recheck starttime after
# opening to close the discovery -> open PID-reuse window.
with suppress(ProcessLookupError, FileNotFoundError):
fd = os.pidfd_open(pid)
try:
current = process_table().get(pid)
if current is not None and current[0] == row[0]:
signal.pidfd_send_signal(fd, sig)
finally:
os.close(fd)
def reap(self, child):
child.poll()
for pid, row in self.discover().items():
if pid != child.pid and row[1] == self.supervisor[0] and row[3] == 'Z':
# An unreaped child cannot reuse its PID. Never reap unrelated jobs.
with suppress(ChildProcessError, ProcessLookupError):
os.waitpid(pid, os.WNOHANG)
def cleanup_group(child, deadline, b, owned, pump=lambda: None):
sent = set()
term_until = min(deadline - 10, b.boottime() + 110)
while b.boottime() < deadline:
sig = signal.SIGTERM if b.boottime() < term_until else signal.SIGKILL
for pid, row in owned.discover().items():
key = (pid, row[0], sig)
if row[3] != 'Z' and key not in sent:
owned.signal(pid, row, sig)
sent.add(key)
owned.reap(child)
pump() # keep draining even when handshake failed or descendants hold pipe
if not owned.discover():
return not owned.unknown
time.sleep(.05)
owned.reap(child)
pump()
return not owned.discover() and not owned.unknown
class StdoutCapture:
"""Nonblocking capture from spawn through cleanup; overflow drains/discards.
Each of three launches has a fixed 1 MiB allowance withheld from all other
writers. This avoids a check/write race with the worker's pending trace bytes.
The 16 MiB cleanup reserve is separate and never consumed by stdout.
"""
def __init__(self, pipe, path, b):
self.pipe, self.b = pipe, b
self.log = Path(path).open('xb', buffering=0) # noqa: SIM115 - owned until cleanup close
try:
os.set_blocking(pipe.fileno(), False)
except BaseException:
try:
self.log.close()
finally:
pipe.close()
raise
self.written = self.discarded = 0
self.eof = False
self.error = None
def pump(self):
# Bounded work per turn, even for an unlimited producer.
for _ in range(16):
try:
block = os.read(self.pipe.fileno(), 65536)
except BlockingIOError:
break
if not block:
self.eof = True
break
retained = block[:max(0, self.b.STDOUT_BYTES - self.written)]
try:
if retained and self.error is None:
if self.log.write(retained) != len(retained):
raise OSError('storage_short_stdout_write')
self.written += len(retained)
except OSError as error:
self.error = error
self.discarded += len(block) - (len(retained) if self.error is None else 0)
return self.discarded
def check(self):
self.pump()
if self.error is not None:
raise self.error
if self.discarded:
raise RuntimeError('storage_stdout_capture_overflow')
def close(self):
try:
self.pump()
os.fsync(self.log.fileno())
finally:
self.log.close()
self.pipe.close()
def exchange_worker(child, channel, payload, budget, capture, owned, monitor, b):
deadline = min(b.boottime() + 30, budget.reservation['deadline_boottime'] - b.CLEANUP_SECONDS)
channel.setblocking(False)
hello = b''
while b'\n' not in hello:
capture.check()
budget.check()
monitor.check()
owned.discover()
if b.boottime() >= deadline:
raise RuntimeError('identity_worker_handshake_timeout')
try:
block = channel.recv(64 - len(hello))
if not block:
raise RuntimeError('identity_worker_handshake_eof')
hello += block
except BlockingIOError:
if child.poll() is not None:
capture.pump()
raise RuntimeError('identity_worker_exited_without_handshake')
time.sleep(.01)
if len(hello) >= 64:
raise RuntimeError('identity_worker_handshake_oversized')
worker_pid = int(hello.strip())
if (os.getpgid(worker_pid) != child.pid or worker_pid not in owned.discover()
or not ancestor(os.getpid(), worker_pid)):
raise RuntimeError('identity_foreign_worker_handshake')
channel.settimeout(max(.01, deadline - b.boottime()))
channel.sendall(b.encode(payload))
channel.shutdown(socket.SHUT_WR)
channel.close()
def supervise(args):
b = cpu_module('evaluation_budget')
policy = storage_policy_check(args, b)
if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'):
raise RuntimeError('process_identity_safe_pidfd_supervisor_required')
grant = grant_check(json.loads(read_trusted(args.grant, args.trusted_grant_sha256)), args, b)
profile = resource_profile(args.protocol)
slots, _, _ = b.profile_limits(profile)
if policy['not_after_boottime'] < max(grant['not_after_boottime'],
b.boottime() + slots[grant['launch_index']][2]
+ (b.CLEANUP_SECONDS if profile is not None else 0)):
raise RuntimeError('storage_policy_expires_before_grant')
owned = OwnedProcesses() # refuse pre-existing jobs before any reservation
monitor = NativeMonitor(policy, b)
environment = dict(os.environ, PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', PYTHONDONTWRITEBYTECODE='1')
launch_command([], environment) # qualify installed launch path before debit
ledger = b.BudgetLedger(args.artifact_root, grant['campaign_id'], cap_bytes=policy['limit_bytes'],
resource_profile=profile)
child = reservation = channels = capture = None
clean = eligible = False
counts = None
errors = []
handlers = {}
monitor_failed = False
def cleanup_pump():
nonlocal monitor_failed
if capture is not None:
capture.pump()
try:
monitor.check()
except Exception as error:
if not monitor_failed:
errors.append(external_failure('supervisor', error))
monitor_failed = True # keep signaling/reaping even after native stop
try:
reservation = ledger.reserve(grant, enforcement_deadline=policy['not_after_boottime'])
output = args.artifact_root / f'launch-{reservation["index"]}'
budget = b.LaunchBudget(args.artifact_root, reservation)
b.durable_directory(output)
handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True))
for s in (signal.SIGTERM, signal.SIGINT)}
budget.write(output / 'reservation.json', reservation)
if profile is not None:
budget.write(output / 'adopted-p1-evidence.json', cpu_module('evaluation_adoption').verify_adoption(
args.artifact_root, grant['p1_adoption_sha256']))
budget.write(output / 'storage-policy.json', monitor.report())
budget.write(output / 'launch-identity.json', launch_identity())
budget.write(output / 'supervisor-identity.json', {
'pid': owned.supervisor[0], 'starttime': owned.supervisor[1],
'dedicated_sole_spawner': True, 'preexisting_children': []})
for name, path in (('manifest.json', args.manifest), ('inventory.json', args.inventory),
('grant.json', args.grant)):
data = Path(path).read_bytes()
budget.check(pending=len(data))
b.durable_write(output / name, data)
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
raise RuntimeError('process_subreaper_setup_failed')
channels = socket.socketpair()
parent, child_channel = channels
child_channel.set_inheritable(True)
native = args.artifact_root / 'native'
if native != native.resolve():
raise RuntimeError('storage_native_redirect_symlink')
if not native.exists():
b.durable_directory(native)
native = native / f'launch-{reservation["index"]}'
b.durable_directory(native)
redirects = native_redirects(native)
for directory in redirects.values():
b.durable_directory(Path(directory))
environment.update(redirects)
command = launch_command([str(ROOT / SCRIPT), '--_worker-fd', str(child_channel.fileno())], environment)
budget.check()
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, start_new_session=True,
pass_fds=(child_channel.fileno(),))
try:
owned.attach(child) # first operation after spawn; capture setup can fail
except BaseException:
# Sole-spawner discovery must still collect children if birth lookup
# failed; unknown ownership can never be reported as clean.
owned.unknown = True
raise
child_channel.close()
capture = StdoutCapture(child.stdout, output / 'stdout.log', b)
arguments = {'protocol': args.protocol, 'device': args.device,
'artifact_root': str(args.artifact_root), 'native_output': str(native),
'inventory': str(output / 'inventory.json'), 'manifest': str(output / 'manifest.json'),
'trusted_manifest_sha256': args.trusted_manifest_sha256}
exchange_worker(child, parent, {'arguments': arguments, 'reservation': reservation,
'launch_identity': launch_identity()},
budget, capture, owned, monitor, b)
while not capture.eof or child.poll() is None:
capture.check()
budget.check()
monitor.check()
owned.discover()
time.sleep(.01)
child.wait(timeout=max(.1, reservation['deadline_boottime'] - b.boottime() - 120))
if (output / 'result.json').is_file():
result = json.loads((output / 'result.json').read_bytes())
counts = result['observed_counts']
postexit = verify_filesystem_inputs(json.loads((output / 'preclose-filesystem.json').read_bytes()))
budget.write(output / 'postexit-filesystem.json', postexit)
eligible = (child.returncode == 0 and result['continuation_eligible']
and result.get('preclose_identity_verified') is True)
except BaseException as error:
errors.append(external_failure('supervisor', error))
finally:
try:
try:
if channels:
for channel in channels:
channel.close()
if child is not None and capture is None and child.stdout is not None:
child.stdout.close() # constructor/attach failure left no capture owner
finally:
if child is not None:
clean = cleanup_group(child, reservation['deadline_boottime'], b, owned, cleanup_pump)
if capture is not None:
capture.close()
if capture.discarded or capture.error:
eligible = False
errors.append(external_failure('recording', RuntimeError('storage_stdout_capture_incomplete')))
if reservation is not None:
if profile is not None:
try:
adoption = cpu_module('evaluation_adoption').verify_adoption(
args.artifact_root, grant['p1_adoption_sha256'])
budget.write(output / 'postexit-adoption.json', {
'p1_adoption_sha256': grant['p1_adoption_sha256'], 'status': 'PASS',
'rows': [{'trial': row['trial'], 'first_failure': row['first_failure'],
'functional_passed': False, 'accepted_skill': False}
for row in adoption['rows']],
'historical_reserved_calls': adoption['historical_reserved_calls'], 'local_reserved_calls': 91000,
'cumulative_reserved_calls': adoption['historical_reserved_calls'] + 91000})
except BaseException as error:
eligible = False
errors.append(external_failure('identity', error))
final_size = b.artifact_size(args.artifact_root, exclude_native=True)
eligible = eligible and not errors and final_size < policy['limit_bytes'] - b.HEADROOM_BYTES
budget.write(output / 'supervisor.json', {
'cleanup_clean': clean, 'errors': errors, 'returncode': None if child is None else child.returncode,
'artifact_bytes_before_finalization': final_size, 'continuation_eligible': eligible and clean,
'storage': monitor.report(), 'stdout_discarded_bytes': capture.discarded if capture else 0,
'stdout_eof_observed': capture.eof if capture else False,
'owned_process_identities': sorted(owned.known), 'ownership_unknown': owned.unknown,
'reserved_physical_calls': reservation['reserved_physical_calls'], 'observed_counts': counts})
ledger.finish(counts=counts, clean=clean, eligible=eligible)
finally:
ledger.close()
for sig, handler in handlers.items():
signal.signal(sig, handler)
return 0 if eligible and clean else 1
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
modes = parser.add_mutually_exclusive_group(required=True)
modes.add_argument('--check-only', action='store_true')
modes.add_argument('--prepare-binding', action='store_true')
modes.add_argument('--run', action='store_true')
modes.add_argument('--_worker-fd', type=int, help=argparse.SUPPRESS)
parser.add_argument('--protocol', choices=PROTOCOLS, default=PROTOCOLS[0])
parser.add_argument('--trial-index', type=int, default=0)
parser.add_argument('--device', default='cuda:0')
parser.add_argument('--inventory', type=Path)
parser.add_argument('--discover-inventory', action='store_true',
help='Preparation only: propose actual imported inventory; never grants execution trust')
parser.add_argument('--output', type=Path)
parser.add_argument('--artifact-root', type=Path)
parser.add_argument('--manifest', type=Path)
parser.add_argument('--trusted-manifest-sha256')
parser.add_argument('--grant', type=Path)
parser.add_argument('--trusted-grant-sha256')
parser.add_argument('--storage-policy', type=Path)
parser.add_argument('--trusted-storage-policy-sha256')
args = parser.parse_args(argv)
if args.discover_inventory and (not args.prepare_binding or args.inventory is not None):
parser.error('--discover-inventory requires preparation only, without --inventory')
if args._worker_fd is not None:
return run_worker(worker_payload(args._worker_fd))
if args.check_only:
protocol = cpu_module('evaluation_protocol').load_protocol(ROOT / args.protocol)
protocol.trial(args.trial_index)
print(json.dumps({'scope': 'CPU protocol/source planning only; no binding, authorization or physical pass',
'stage': protocol.document['stage'], 'rows': [asdict(t) for t in protocol.trials],
'future_sources': FUTURE_SOURCES,
'resource_profile': resource_profile(args.protocol),
'reserved_physical_calls': [slot[0] for slot in
cpu_module('evaluation_budget').profile_limits(resource_profile(args.protocol))[0]],
'actual_backend_imports_and_performance': 'unqualified'}))
return 0
if args.prepare_binding:
return prepare(args)
if any(getattr(args, name) is None for name in ('inventory', 'artifact_root', 'manifest',
'trusted_manifest_sha256', 'grant', 'trusted_grant_sha256')):
parser.error('--run requires reviewed inventory, artifact root, manifest+trusted digest, '
'fresh grant+trusted digest')
return supervise(args)
if __name__ == '__main__':
raise SystemExit(main())