9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
1948 lines
101 KiB
Python
1948 lines
101 KiB
Python
#!/usr/bin/env python3
|
|
"""Prospective P1/P2 evaluator. CPU --check-only; Kit --prepare-binding; gated --run.
|
|
|
|
No mode grants its own permission. --run is a Linux supervisor, not a direct Kit
|
|
invocation. It consumes a fresh independently trusted parent grant and binding.
|
|
Never use an old replay grant. See logs/contact_cycle/evaluation_runner_v1/README.md
|
|
and docs/contact_cycle_native_monitor_v3.md (superseding native coverage policy).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import ctypes
|
|
import dataclasses
|
|
import enum
|
|
import errno
|
|
import functools
|
|
import hashlib
|
|
import importlib
|
|
import importlib.util
|
|
import inspect
|
|
import json
|
|
import math
|
|
import os
|
|
import shlex
|
|
import signal
|
|
import socket
|
|
import stat
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from contextlib import ExitStack, suppress
|
|
from dataclasses import asdict
|
|
from pathlib import Path, PosixPath, WindowsPath
|
|
from types import GetSetDescriptorType, MappingProxyType, MemberDescriptorType, ModuleType, SimpleNamespace
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SCRIPT = 'scripts/evaluate_cycle_sim_residual.py'
|
|
PACKAGE = 'source/SkillFactory/SkillFactory/'
|
|
TASK = 'SkillFactory.tasks.manager_based.contact_cycle_sim.'
|
|
LEG = 'SkillFactory.legs.contact_cycle_sim.'
|
|
PROTOCOLS = ('configs/contact_cycle/continuous_eval_p1_v1.json',
|
|
'configs/contact_cycle/bounded_diagnostic_p2_v1.json',
|
|
'configs/contact_cycle/bounded_diagnostic_p2_extended_v1.json')
|
|
FUTURE_SOURCES = {
|
|
'runner': [SCRIPT, PACKAGE + 'legs/contact_cycle_sim/evaluation_budget.py'],
|
|
'instrumentation': [PACKAGE + 'tasks/manager_based/contact_cycle_sim/' + name + '.py'
|
|
for name in ('evaluation_instrumentation', 'evaluation_rng_scope')],
|
|
}
|
|
REAL_TEST = 'tests/test_contact_cycle_sim_replay.py::test_actual_frozen_candidate_cpu_only_when_artifacts_present'
|
|
LAUNCHER = '/home/cen/IsaacLab/isaaclab.sh'
|
|
OPTIONAL_REPLICATOR = 'omni.replicator.core'
|
|
|
|
# Exact constructor-late imports evidenced in p1_first_failure_diagnosis.md.
|
|
# Full source identities are checked before import and retained in the binding.
|
|
CONSTRUCTOR_IMPORT_PINS = {
|
|
'isaaclab.cloner.cloner_cfg': (
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cloner/cloner_cfg.py',
|
|
'f48e268609e8e5e91e30b8e6d364cc76579aeb497d69f963cd45975152a282ca'),
|
|
'isaaclab.cloner.cloner_strategies': (
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cloner/cloner_strategies.py',
|
|
'ea5943bb5e17df09b792dcf7de436f7481daed8768dc2d8573c6eb45464481d8'),
|
|
'isaaclab.sim.schemas.schemas_actuators': (
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/schemas/schemas_actuators.py',
|
|
'3defa9a6ff41d39e797c6d4a1dfee031913fdaf64ff7f99305a844f4ddf47f77'),
|
|
'isaaclab.sim.spawners.wrappers': (
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/spawners/wrappers/__init__.py',
|
|
'431dfebf1f14464b5479fe618720d913fdefb8e1d613a0c54f45dcddb671cce5'),
|
|
'isaaclab.sim.spawners.wrappers.wrappers_cfg': (
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/spawners/wrappers/wrappers_cfg.py',
|
|
'ce073f03584f9cab400735fce1af92b261b0e9217592c7ce79c1b8f541399574'),
|
|
'isaaclab.sim.views.usd_frame_view': (
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/sim/views/usd_frame_view.py',
|
|
'6c186fe2991a608323f12f40fb67516dd629079eabe4f6e525c198e0452a59d9'),
|
|
'isaaclab.utils.warp.fabric': (
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/utils/warp/fabric.py',
|
|
'3756853121fbd0fb88b01f07c521927e9cdc1ccd96ceb40b6a603bef29e56c94'),
|
|
'isaaclab_physx.assets.surface_gripper': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/assets/surface_gripper/__init__.py',
|
|
'42992defc933ea096bac4666c574bfd32ace19fb3886f6e41b5d5498e06da2ef'),
|
|
'isaaclab_physx.assets.surface_gripper.surface_gripper_cfg': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/assets/surface_gripper/surface_gripper_cfg.py',
|
|
'c71846dad68db328c6faa9ac7dfe37488502e9477f56de0961ab3648f273b14a'),
|
|
'isaaclab_physx.cloner': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/__init__.py',
|
|
'13e3966a049c8a6feeee9a8018162ddce6667f9cbdc0d7350644b2176a191def'),
|
|
'isaaclab_physx.cloner.physx_replicate': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/physx_replicate.py',
|
|
'a0e7eb7f47e45ce27351002a0ef6c137ddb3c1c4a20a5efcbbd389c3ff5a426c'),
|
|
'isaaclab_physx.sensors': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/__init__.py',
|
|
'52dc0bed9f1cc96b938b9a6aae4f2f48076006fce0663c2e8ad017488f8098a4'),
|
|
'isaaclab_physx.sensors.contact_sensor': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/__init__.py',
|
|
'2f81a5de0c1a2a264fc18832fbfe33e3cb49a41a7a246b0de4471437dcbf3dfd'),
|
|
'isaaclab_physx.sensors.contact_sensor.contact_sensor': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py',
|
|
'09507db17049bcadd6a48f5992bc20f79b019356c13f1ba3ef1527b2e4188b2c'),
|
|
'isaaclab_physx.sensors.contact_sensor.contact_sensor_data': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor_data.py',
|
|
'711c461be5c696a7a99f0a8fd2a010cd9a5a47e1b8bcfdf493e5ac8af8b65e3d'),
|
|
'isaaclab_physx.sensors.contact_sensor.kernels': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/kernels.py',
|
|
'e3e9d678336a704a9ba58fe7d31fe8a3a577f66f357c921a2c312c2edb8fa6e3'),
|
|
'isaaclab_physx.sensors.joint_wrench': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/__init__.py',
|
|
'308cb956e92e065c05b66ff0ec0c139dc3bc35380a09d58191af13acf5c14f36'),
|
|
'isaaclab_physx.sensors.joint_wrench.joint_wrench_sensor': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/joint_wrench_sensor.py',
|
|
'eff63d6f6e8c9958651280889c1797c9cbf2a14fb1b35a1f4e39cb208399ee5f'),
|
|
'isaaclab_physx.sensors.joint_wrench.joint_wrench_sensor_data': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/joint_wrench_sensor_data.py',
|
|
'32b168096dc384587e2962b8e09caccba1d76bb8e40459073b262843b6511937'),
|
|
'isaaclab_physx.sensors.joint_wrench.kernels': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/joint_wrench/kernels.py',
|
|
'a71e33ae91ab7bef9d585d7ca8137028a3dfe5b169dabec4b9b763f8497d25a0'),
|
|
'isaaclab_physx.sensors.kernels': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sensors/kernels.py',
|
|
'297f79677d58d7c5ed38cffc7ba19480bc3236a331a64700a54d57f34a5b8b56'),
|
|
'isaaclab_physx.sim.views': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sim/views/__init__.py',
|
|
'ac57ce5aafa903fe42868a410461b9db8fc8a150ed458f9b8bfe80772a94f97c'),
|
|
'isaaclab_physx.sim.views.fabric_frame_view': (
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/sim/views/fabric_frame_view.py',
|
|
'7d9d199be7dc8b9087c60f9f89567fb0463fd0269d20b0aa0e20ae7f8ac1d329'),
|
|
}
|
|
|
|
|
|
# One reviewed installed shell/CLI layout only; the bridge changes subprocess.run
|
|
# to exec, not the inherited socket authentication or the runtime import tree.
|
|
LAUNCH_PINS = {
|
|
'/home/cen/IsaacLab/isaaclab.sh':
|
|
'c3a6f3111012aa39c5f568012da1ef2cde52a80cbc2277bc59ce295895d7bc68',
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/__init__.py':
|
|
'8886104cee16eb41c8c7dccbe2955cf8e4c72677f49423bf586f4eb4615a96af',
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/envs.py':
|
|
'92002ebc7301f559cd17dcbf13a749006c1353aabf9f1247e57a9438a85acecd',
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/format.py':
|
|
'bc575ef81dff01a4c8a7b11496515ac34fa9bbdf14613978e84fe37449cd64cc',
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/install.py':
|
|
'b285da1f19beada43c1a4ad9aa7e3b432a0042eb0074c7c89ccc7aa0d3c46b31',
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/commands/misc.py':
|
|
'b608ffecbae98c810024dc361852325d09197a9f9a355b270bd60c410a459747',
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/cli/utils.py':
|
|
'9608423230d159234e740a8000fd8206a4c980311e6517f97d785eb58f68fc97',
|
|
'/home/cen/IsaacLab/source/isaaclab/isaaclab/__init__.py':
|
|
'24e50c4e57b9be1c55aa949f1b8dd416d95e6002e12941abe33a7a6eb16f697b',
|
|
'/home/cen/isaacsim-6.0.1/python.sh':
|
|
'37f7e3717f1fce1800d0b15b55fcd754c909978bd2455b9d6de3d22d77a92bc3',
|
|
'/home/cen/isaacsim-6.0.1/setup_python_env.sh':
|
|
'0f73e6dac65efa1a28f994939f4734859d2c39eb236c658bfad9fab70c8882e6',
|
|
'/home/cen/isaacsim-6.0.1/kit/python/bin/python3':
|
|
'f3300e753c44afd7860b3f26ccf7c3d98c323b4fc90e7ce2fccae2cd49efd09c',
|
|
'/home/cen/isaacsim-6.0.1/kit/libcarb.so':
|
|
'aef0445bb75bd3da04acfabf6d067fb9e40b5c3d17f07dd5369bd63c75489c4f',
|
|
'/usr/bin/python3.12':
|
|
'e50d468e8b0adfb05733f5b87b3cff34829c4a8c1aea50c865aa8bdfe4bb150f',
|
|
'/usr/bin/bash':
|
|
'bc5945feb8bd26203ebfafea5ce1878bb2e32cb8fb50ab7ae395cfb1e1aaaef1',
|
|
}
|
|
|
|
RESOLVED_PYTHON = '/home/cen/isaacsim-6.0.1/kit/python/bin/python3'
|
|
EXEC_BRIDGE = """import os, sys
|
|
from pathlib import Path
|
|
from isaaclab.cli.utils import extract_python_exe
|
|
resolved = extract_python_exe()
|
|
if Path(resolved).resolve() != Path('/home/cen/isaacsim-6.0.1/python.sh'):
|
|
raise RuntimeError('identity_unreviewed_resolved_interpreter')
|
|
os.execv(resolved, [resolved, *sys.argv[1:]])
|
|
"""
|
|
|
|
|
|
def launch_identity():
|
|
b = cpu_module('evaluation_budget')
|
|
for path, expected in LAUNCH_PINS.items():
|
|
if b.digest(path) != expected:
|
|
raise RuntimeError('identity_launch_source_changed: ' + path)
|
|
# This layout has no optional setup_conda_env.sh or docker startup helper.
|
|
sim = Path('/home/cen/IsaacLab/_isaac_sim')
|
|
if (sim.resolve() != Path('/home/cen/isaacsim-6.0.1')
|
|
or (sim / 'setup_conda_env.sh').exists() or (sim / 'vulkan_check.sh').exists()
|
|
or any(Path('/home/cen/IsaacLab', name).exists() for name in ('env_isaaclab', '.venv'))):
|
|
raise RuntimeError('identity_unreviewed_launch_layout')
|
|
return {'format': 'skillfactory.fd-exec-launch.v1', 'sources': LAUNCH_PINS,
|
|
'bridge_sha256': hashlib.sha256(EXEC_BRIDGE.encode()).hexdigest(),
|
|
'runner_sha256': b.digest(ROOT / SCRIPT), 'interpreter': RESOLVED_PYTHON}
|
|
|
|
|
|
def launch_command(arguments, environment):
|
|
launch_identity()
|
|
if any(environment.get(k) for k in ('VIRTUAL_ENV', 'CONDA_PREFIX', 'PYTHONEXE', 'PYTHONHOME', 'args')):
|
|
raise RuntimeError('identity_launch_requires_reviewed_no_virtualenv_layout')
|
|
text = Path(LAUNCHER).read_text()
|
|
text = text.replace('export ISAACLAB_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"',
|
|
'export ISAACLAB_PATH=/home/cen/IsaacLab')
|
|
old = 'exec "$python_exe" -c "from isaaclab.cli import cli; cli()" "$@"'
|
|
if text.count(old) != 1:
|
|
raise RuntimeError('identity_launch_exec_boundary_changed')
|
|
text = text.replace(old, 'exec "$python_exe" -c ' + shlex.quote(EXEC_BRIDGE) + ' "$@"')
|
|
return ['/usr/bin/bash', '-c', text, LAUNCHER, *arguments]
|
|
|
|
|
|
def cpu_module(name):
|
|
"""Load pure stdlib source without importing Isaac registration before Kit.
|
|
|
|
Budget has its real file-backed module identity (no fake parent packages).
|
|
CPU-only protocol planning uses a private identity, never a binding snapshot.
|
|
"""
|
|
if name == 'evaluation_adoption':
|
|
cpu_module('evaluation_budget')
|
|
full = LEG + name if name in ('evaluation_budget', 'evaluation_adoption') else '_evaluation_cpu_' + name
|
|
if full not in sys.modules:
|
|
path = ROOT / PACKAGE / 'legs/contact_cycle_sim' / (name + '.py')
|
|
spec = importlib.util.spec_from_file_location(full, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[full] = module
|
|
spec.loader.exec_module(module)
|
|
return sys.modules[full]
|
|
|
|
|
|
def protect_artifact_path(path):
|
|
path = Path(path)
|
|
allowed = ROOT / 'logs/contact_cycle/evaluation_runs_v1'
|
|
if (not path.is_absolute() or path != path.resolve()
|
|
or (path.is_relative_to(ROOT) and not path.is_relative_to(allowed))):
|
|
raise ValueError('Use new evaluation_runs_v1 campaign paths or canonical paths outside repository; '
|
|
'historic evidence/source/release trees are protected')
|
|
|
|
|
|
def read_trusted(path, expected):
|
|
payload = Path(path).read_bytes()
|
|
if (len(expected or '') != 64 or any(c not in '0123456789abcdef' for c in expected)
|
|
or hashlib.sha256(payload).hexdigest() != expected):
|
|
raise ValueError('identity_independently_trusted_digest_required: ' + str(path))
|
|
return payload
|
|
|
|
|
|
def resource_profile(protocol_path):
|
|
if protocol_path not in PROTOCOLS:
|
|
raise ValueError('Unknown fixed protocol path')
|
|
protocol = cpu_module('evaluation_protocol').load_protocol(ROOT / protocol_path)
|
|
return protocol.document.get('resource_profile')
|
|
|
|
|
|
def protocol_stage(protocol_path):
|
|
"""Fixed-path stage lookup, safe before real runtime binding.
|
|
|
|
Never load a private CPU planning module into the Kit worker. Full protocol
|
|
validation remains with the supervisor and the real runtime module.
|
|
"""
|
|
if protocol_path not in PROTOCOLS:
|
|
raise ValueError('Unknown fixed protocol path')
|
|
return 'P1' if protocol_path == PROTOCOLS[0] else 'P2'
|
|
|
|
|
|
def grant_check(grant, args, b):
|
|
extended = resource_profile(args.protocol) == b.EXTENDED_PROFILE
|
|
fields = {'format', 'campaign_id', 'launch_index', 'stage', 'trial_indices', 'manifest_sha256',
|
|
'protocol_sha256', 'inventory_sha256', 'artifact_root', 'boot_id',
|
|
'not_after_boottime', 'parent_review', 'p1_evidence_sha256', 'storage_policy_sha256',
|
|
'storage_scope', 'launch_identity_sha256'}
|
|
if extended:
|
|
fields = (fields - {'p1_evidence_sha256'}) | {'resource_profile', 'p1_adoption_sha256'}
|
|
if not isinstance(grant, dict) or set(grant) != fields:
|
|
raise ValueError('Explicit fresh parent grant schema required')
|
|
index = grant['launch_index']
|
|
if type(index) is not int or index not in ((0,) if extended else (0, 1, 2)):
|
|
raise ValueError('Parent grant launch slot')
|
|
expected_stage = 'P2' if extended or index == 2 else 'P1'
|
|
rows = list(range(20)) if expected_stage == 'P2' else [index]
|
|
if (grant['format'] != (b.EXTENDED_GRANT_FORMAT if extended else 'skillfactory.evaluation-parent-grant.v2')
|
|
or not isinstance(grant['campaign_id'], str) or not grant['campaign_id']
|
|
or not isinstance(grant['parent_review'], str) or not grant['parent_review'].strip()
|
|
or grant['stage'] != expected_stage or grant['trial_indices'] != rows
|
|
or grant['manifest_sha256'] != args.trusted_manifest_sha256
|
|
or grant['protocol_sha256'] != b.digest(ROOT / args.protocol)
|
|
or grant['inventory_sha256'] != b.digest(args.inventory)
|
|
or grant['storage_policy_sha256'] != args.trusted_storage_policy_sha256
|
|
or grant['storage_scope'] != b.STORAGE_SCOPE
|
|
or grant['launch_identity_sha256'] != hashlib.sha256(b.encode(launch_identity())).hexdigest()
|
|
or grant['artifact_root'] != str(args.artifact_root)
|
|
or grant['boot_id'] != b.boot_id()
|
|
or args.protocol != PROTOCOLS[2 if extended else index == 2]
|
|
or type(grant['not_after_boottime']) not in (int, float)
|
|
or not b.boottime() < grant['not_after_boottime'] <= b.boottime() + 3600
|
|
or (not extended and index < 2 and grant['p1_evidence_sha256'] != [])
|
|
or (not extended and index == 2 and (not isinstance(grant['p1_evidence_sha256'], list)
|
|
or len(grant['p1_evidence_sha256']) != 2))):
|
|
raise ValueError('Parent grant identity/order/expiry/review mismatch')
|
|
if extended:
|
|
if grant['resource_profile'] != b.EXTENDED_PROFILE or args.artifact_root not in (b.EXTENDED_ROOT, b.SECOND_EXTENDED_ROOT):
|
|
raise ValueError('Parent grant fixed extended profile/root mismatch')
|
|
adoption = cpu_module('evaluation_adoption').verify_adoption(args.artifact_root, grant['p1_adoption_sha256'])
|
|
if adoption['source_boot_id'] != b.boot_id():
|
|
raise ValueError('Parent grant adoption same-boot mismatch')
|
|
read_trusted(args.manifest, args.trusted_manifest_sha256)
|
|
return grant
|
|
|
|
|
|
def mount_identity(path):
|
|
"""Read-only mount facts, NOT evidence that a quota or confinement is effective."""
|
|
matches = []
|
|
for line in Path('/proc/self/mountinfo').read_text().splitlines():
|
|
fields = line.split()
|
|
mount = Path(fields[4].replace('\\040', ' '))
|
|
if path == mount or mount in path.parents:
|
|
matches.append((len(mount.parts), line))
|
|
if not matches:
|
|
raise RuntimeError('storage_mount_identity_unavailable')
|
|
return max(matches)[1]
|
|
|
|
|
|
STORAGE_APPROVAL = 'logs/contact_cycle/evaluation_runner_recovery_v1/USER_STORAGE_SCOPE_AMENDMENT.md'
|
|
STORAGE_APPROVAL_SHA256 = 'e4fac0d36799c9beb67af7687a390a6b5e8390bde1d4b840a6e26c309578c8db'
|
|
NATIVE_GAPS = ('sampled logical sizes, not a quota; writes can overshoot between checks',
|
|
'unlisted/global cache writers, deleted-open files, symlink targets and native allocations '
|
|
'have unknown bytes/growth; only free-space floors on listed filesystems cover external writes',
|
|
'shared native cache changes include unrelated jobs; growth can cause conservative stops',
|
|
'stable non-runtime-owned inaccessible shared subtrees are explicit permission gaps; '
|
|
'their bytes and growth are unknown; coverage is incomplete',
|
|
'generic HOME/.cache and /tmp are not recursively scanned; unlisted or hidden mounts are uncovered',
|
|
'installed data and kit/data/cache/logs are conservative portable-token candidates; '
|
|
'actual token destinations require runtime confirmation; missing candidates are not created')
|
|
SIM_ROOT = Path('/home/cen/isaacsim-6.0.1')
|
|
|
|
|
|
def native_paths(root):
|
|
# SimulationApp --portable + kit-core.json data/log writers; token resolution
|
|
# remains unqualified. Never substitute a generic home/cache/tmp tree.
|
|
return sorted({str(root / 'native'), *shared_native_paths(),
|
|
*[str(SIM_ROOT / name) for name in ('data', 'kit/data', 'kit/logs')]})
|
|
|
|
|
|
def shared_native_paths():
|
|
# Installed clear_caches.sh names these three ov caches explicitly.
|
|
return sorted({*[str(Path.home() / '.cache/ov' / name)
|
|
for name in ('shaders', 'texturecache', 'Kit/107.3')], str(SIM_ROOT / 'kit/cache')})
|
|
|
|
|
|
def free_space_paths(root):
|
|
# Other native roots also observe their own filesystem during their scan.
|
|
return sorted({str(root), '/tmp', str(Path.home()), str(SIM_ROOT)})
|
|
|
|
|
|
def native_redirects(native, *, preparation=False):
|
|
redirects = {'HOME': str(native / 'home'), 'XDG_CACHE_HOME': str(native / 'cache'),
|
|
'TMPDIR': str(native / 'tmp')}
|
|
if preparation:
|
|
redirects.update(XDG_CONFIG_HOME=str(native / 'config'), XDG_DATA_HOME=str(native / 'data'))
|
|
return redirects
|
|
|
|
|
|
def task_owned_cache_paths(root):
|
|
# Same mapping used to construct the actual subprocess environments.
|
|
native = root / 'native'
|
|
return sorted([*native_redirects(native / 'preparation-support', preparation=True).values(),
|
|
*[path for index in range(3) for path in native_redirects(native / f'launch-{index}').values()]])
|
|
|
|
|
|
def storage_policy_check(args, b):
|
|
if args.storage_policy is None or args.trusted_storage_policy_sha256 is None:
|
|
raise RuntimeError('storage_bound_policy_unavailable; no launch permitted')
|
|
extended = resource_profile(getattr(args, 'protocol', PROTOCOLS[0])) == b.EXTENDED_PROFILE
|
|
policy = json.loads(read_trusted(args.storage_policy, args.trusted_storage_policy_sha256))
|
|
read_trusted(ROOT / STORAGE_APPROVAL, STORAGE_APPROVAL_SHA256)
|
|
fields = {'format', 'artifact_root', 'limit_bytes', 'scope', 'approval_sha256',
|
|
'native_paths', 'check_interval_seconds', 'max_native_growth_bytes', 'min_free_bytes',
|
|
'coverage_gaps', 'native_hard_cap', 'boot_id', 'mount_identity', 'not_after_boottime',
|
|
'shared_native_paths', 'task_owned_cache_paths', 'free_space_paths', 'free_space_mount_identities'}
|
|
root = args.artifact_root
|
|
protect_artifact_path(root)
|
|
if extended:
|
|
fields |= {'resource_profile', 'issued_boottime'}
|
|
if (root not in (b.EXTENDED_ROOT, b.SECOND_EXTENDED_ROOT) or policy.get('resource_profile') != b.EXTENDED_PROFILE
|
|
or type(policy.get('issued_boottime')) not in (int, float)
|
|
or not 0 <= policy['issued_boottime'] <= b.boottime()
|
|
or type(policy.get('not_after_boottime')) not in (int, float)
|
|
or not policy['issued_boottime'] < policy['not_after_boottime']
|
|
<= policy['issued_boottime'] + b.EXTENDED_TOTAL_SECONDS):
|
|
raise RuntimeError('storage_extended_profile_lifetime_mismatch')
|
|
if (set(policy) != fields or policy['format'] != (
|
|
b.EXTENDED_POLICY_FORMAT if extended else 'skillfactory.output-storage-policy.v4')
|
|
or not root.is_dir() or policy['artifact_root'] != str(root)
|
|
or type(policy['limit_bytes']) is not int
|
|
or not b.HEADROOM_BYTES + b.STDOUT_ALLOWANCE < policy['limit_bytes'] <= b.CAP_BYTES
|
|
or policy['scope'] != b.STORAGE_SCOPE or policy['approval_sha256'] != STORAGE_APPROVAL_SHA256
|
|
or policy['native_paths'] != native_paths(root) or policy['native_hard_cap'] is not False
|
|
or policy['coverage_gaps'] != list(NATIVE_GAPS)
|
|
or policy['shared_native_paths'] != shared_native_paths()
|
|
or policy['task_owned_cache_paths'] != task_owned_cache_paths(root)
|
|
or policy['free_space_paths'] != free_space_paths(root)
|
|
or policy['free_space_mount_identities'] != {p: mount_identity(Path(p)) for p in free_space_paths(root)}
|
|
or any(Path(p).is_relative_to(root) for p in shared_native_paths())
|
|
or type(policy['check_interval_seconds']) not in (int, float)
|
|
or not 0 < policy['check_interval_seconds'] <= 1
|
|
or type(policy['max_native_growth_bytes']) is not int
|
|
or not 0 < policy['max_native_growth_bytes'] <= b.CAP_BYTES
|
|
or type(policy['min_free_bytes']) is not int or policy['min_free_bytes'] < 2 * b.CAP_BYTES
|
|
or policy['boot_id'] != b.boot_id() or policy['mount_identity'] != mount_identity(root)
|
|
or type(policy['not_after_boottime']) not in (int, float)
|
|
or not b.boottime() < policy['not_after_boottime'] <= b.boottime() + (14400 if extended else 3600)):
|
|
raise RuntimeError('storage_policy_identity_or_conditions_mismatch')
|
|
return policy
|
|
|
|
|
|
class NativeScanError(RuntimeError):
|
|
"""Retain partial scan facts even when baseline fails before monitor creation."""
|
|
def __init__(self, report):
|
|
self.report = report
|
|
super().__init__('storage_native_scan_failed: ' + json.dumps(report, sort_keys=True))
|
|
|
|
|
|
def native_metadata(info):
|
|
return {key: getattr(info, 'st_' + key) for key in ('uid', 'gid', 'mode', 'dev', 'ino', 'ctime_ns')}
|
|
|
|
|
|
def native_directory_identity(info):
|
|
# Creating entries changes ctime during normal writes; it is not a new directory.
|
|
return (info.st_dev, info.st_ino, info.st_uid, info.st_gid, info.st_mode)
|
|
|
|
|
|
def native_directory_names(fd, *, recursive):
|
|
if not recursive:
|
|
return []
|
|
with os.scandir(fd) as entries:
|
|
return [entry.name for entry in entries]
|
|
|
|
|
|
def native_size(path, *, shared=False, strict_paths=(), exclude_paths=(), required=False, recursive=True):
|
|
"""Descriptor-relative traversal; never follow even a raced ancestor symlink.
|
|
|
|
Only an actual denied directory operation plus stable non-owned metadata and
|
|
an effective-credential access denial permits a shared coverage gap. A gap's
|
|
contents are not inspected. recursive=False observes only root/ancestor
|
|
filesystem facts, never directory entries. Missing ordinary entries are sampled churn, not
|
|
permission exclusions; disappearance during denial qualification is fatal.
|
|
"""
|
|
path = Path(path)
|
|
strict = tuple(Path(p) for p in strict_paths)
|
|
excluded = set(map(Path, exclude_paths))
|
|
result = {'logical_bytes': 0, 'permission_gaps': [], 'skipped_count': 0,
|
|
'disappeared_paths': [], 'symlink_paths': [], 'free_space': None}
|
|
|
|
def fail(current, operation, error, info=None):
|
|
result['failure'] = {'path': str(current), 'operation': operation,
|
|
'errno': error.errno, 'error': str(error),
|
|
'metadata': native_metadata(info) if info is not None else None}
|
|
raise NativeScanError(result) from error
|
|
|
|
def denied(parent, name, current, info, operation, error):
|
|
# An inaccessible ancestor of a strict path would also hide owned writes.
|
|
protected = any(current.is_relative_to(p) or p.is_relative_to(current) for p in strict)
|
|
if (not shared or protected or error.errno not in (errno.EACCES, errno.EPERM)
|
|
or info.st_uid in (os.getuid(), os.geteuid()) or not stat.S_ISDIR(info.st_mode)):
|
|
fail(current, operation, error, info)
|
|
try:
|
|
after = os.stat(name, dir_fd=parent, follow_symlinks=False)
|
|
accessible = os.access(name, os.R_OK | os.X_OK, dir_fd=parent,
|
|
effective_ids=True, follow_symlinks=False)
|
|
final = os.stat(name, dir_fd=parent, follow_symlinks=False)
|
|
except OSError as metadata_error:
|
|
result['denied_operation'] = {'path': str(current), 'errno': error.errno,
|
|
'metadata': native_metadata(info)}
|
|
fail(current, 'permission_metadata_recheck', metadata_error, info)
|
|
if (native_metadata(info) != native_metadata(after)
|
|
or native_metadata(info) != native_metadata(final) or accessible):
|
|
fail(current, 'permission_identity_or_access_changed', error, final)
|
|
result['permission_gaps'].append({'path': str(current), 'errno': error.errno,
|
|
'operation': operation, **native_metadata(final)})
|
|
result['skipped_count'] += 1
|
|
|
|
def observe_free(fd, current):
|
|
try:
|
|
usage = os.fstatvfs(fd)
|
|
result['free_space'] = {'observed_path': str(current),
|
|
'free_bytes': usage.f_bavail * usage.f_frsize,
|
|
'device': os.fstat(fd).st_dev, 'mount_identity': mount_identity(current)}
|
|
except OSError as error:
|
|
fail(current, 'free_space', error)
|
|
|
|
def visit(parent, name, current, *, ancestor=False):
|
|
if current in excluded:
|
|
return
|
|
try:
|
|
info = os.stat(name, dir_fd=parent, follow_symlinks=False)
|
|
except FileNotFoundError as error:
|
|
if required and (ancestor or current == path):
|
|
fail(current, 'required_directory', error)
|
|
result['disappeared_paths'].append(str(current))
|
|
return
|
|
except OSError as error:
|
|
fail(current, 'lstat', error)
|
|
if stat.S_ISLNK(info.st_mode):
|
|
# Explicit monitored roots/ancestors and owned trees cannot be aliases.
|
|
if ancestor or current == path or any(current.is_relative_to(p) or p.is_relative_to(current)
|
|
for p in strict):
|
|
fail(current, 'symlink', OSError(errno.ELOOP, 'monitored path is a symlink'), info)
|
|
result['symlink_paths'].append(str(current))
|
|
return
|
|
if ((ancestor or current == path or any(p.is_relative_to(current) for p in strict))
|
|
and not stat.S_ISDIR(info.st_mode)):
|
|
fail(current, 'directory', OSError(errno.ENOTDIR, 'not a directory'), info)
|
|
if stat.S_ISREG(info.st_mode):
|
|
if not os.access(name, os.R_OK, dir_fd=parent, effective_ids=True, follow_symlinks=False):
|
|
# access() returns False for ENOENT as well as permission denial.
|
|
# Atomic .pending publication may race the earlier lstat. Only a
|
|
# descriptor-relative ENOENT recheck is ordinary sampled churn;
|
|
# retain its last observed bytes conservatively, never a gap.
|
|
try:
|
|
os.stat(name, dir_fd=parent, follow_symlinks=False)
|
|
except FileNotFoundError:
|
|
result['disappeared_paths'].append(str(current))
|
|
result['logical_bytes'] += info.st_size
|
|
return
|
|
except OSError as error:
|
|
fail(current, 'file_access_recheck', error, info)
|
|
fail(current, 'file_access', PermissionError(None, 'file unreadable; access() exposes no errno'), info)
|
|
result['logical_bytes'] += info.st_size
|
|
return
|
|
if not stat.S_ISDIR(info.st_mode):
|
|
return
|
|
fd = None
|
|
operation = 'open'
|
|
try:
|
|
try:
|
|
fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent)
|
|
operation = 'fstat'
|
|
if native_directory_identity(os.fstat(fd)) != native_directory_identity(info):
|
|
fail(current, 'open_identity_changed', OSError(errno.ESTALE, 'directory changed'), info)
|
|
if ancestor:
|
|
walk_to_root(fd, current)
|
|
else:
|
|
if current == path:
|
|
observe_free(fd, current)
|
|
# Scandir errors belong to this directory, not one of its children.
|
|
operation = 'scandir'
|
|
names = native_directory_names(fd, recursive=recursive)
|
|
operation = 'children'
|
|
for child in names:
|
|
visit(fd, child, current / child)
|
|
operation = 'traversal_lstat'
|
|
after = os.stat(name, dir_fd=parent, follow_symlinks=False)
|
|
if native_directory_identity(after) != native_directory_identity(info):
|
|
fail(current, 'traversal_identity_changed', OSError(errno.ESTALE, 'directory changed'), after)
|
|
except OSError as error:
|
|
if operation in ('open', 'scandir') and error.errno in (errno.EACCES, errno.EPERM) and not ancestor:
|
|
denied(parent, name, current, info, operation, error)
|
|
else:
|
|
fail(current, operation, error, info)
|
|
finally:
|
|
if fd is not None:
|
|
os.close(fd)
|
|
|
|
def walk_to_root(fd, current):
|
|
observe_free(fd, current)
|
|
relative = path.relative_to(current)
|
|
if not relative.parts:
|
|
raise ValueError('native scan root must not be /')
|
|
child = current / relative.parts[0]
|
|
visit(fd, relative.parts[0], child, ancestor=child != path)
|
|
|
|
try:
|
|
with ExitStack() as stack:
|
|
fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
stack.callback(os.close, fd)
|
|
walk_to_root(fd, Path('/'))
|
|
except OSError as error:
|
|
fail(path, 'ancestor', error)
|
|
return result
|
|
|
|
|
|
class NativeMonitor:
|
|
def __init__(self, policy, b):
|
|
self.policy, self.b = policy, b
|
|
self.baseline = self.latest = {}
|
|
self.baseline_scans = self.latest_scans = {}
|
|
self.baseline_free_space = self.latest_free_space = {}
|
|
self.gap_history = []
|
|
self.growth = 0
|
|
self.next_check = 0
|
|
self.latest_scan_completed = False
|
|
try:
|
|
self.check(force=True)
|
|
except Exception as error:
|
|
if not isinstance(error, NativeScanError):
|
|
error.report = self.report()
|
|
raise
|
|
|
|
def check(self, *, force=False):
|
|
now = self.b.boottime()
|
|
if not force and now < self.next_check:
|
|
return
|
|
self.next_check = now + self.policy['check_interval_seconds']
|
|
self.latest_scan_completed = False
|
|
root = Path(self.policy['artifact_root'])
|
|
if mount_identity(root) != self.policy['mount_identity']:
|
|
raise RuntimeError('storage_policy_mount_identity_changed')
|
|
paths = self.policy['native_paths']
|
|
strict = [str(root), *self.policy['task_owned_cache_paths'],
|
|
*[p for p in paths if p not in self.policy['shared_native_paths']]]
|
|
scans = {}
|
|
self.latest_free_space = {}
|
|
checking_free_space = True
|
|
try:
|
|
for path in self.policy['free_space_paths']:
|
|
if mount_identity(Path(path)) != self.policy['free_space_mount_identities'][path]:
|
|
raise RuntimeError('storage_policy_free_space_mount_identity_changed: ' + path)
|
|
observation = native_size(path, required=True, recursive=False)
|
|
self.latest_free_space[path] = observation['free_space']
|
|
if observation['free_space']['mount_identity'] != self.policy['free_space_mount_identities'][path]:
|
|
raise RuntimeError('storage_policy_free_space_mount_identity_changed: ' + path)
|
|
if observation['free_space']['free_bytes'] < self.policy['min_free_bytes']:
|
|
raise RuntimeError('storage_native_free_space_stop')
|
|
checking_free_space = False
|
|
# Disjoint scans: output accessibility is NOT native growth/cap accounting.
|
|
# The unchanged pre-write aggregate cap still excludes only root/native.
|
|
for path in [str(root), *paths]:
|
|
exclusions = [p for p in [str(root), *paths] if p != path and Path(p).is_relative_to(path)]
|
|
scans[path] = native_size(path, shared=path in self.policy['shared_native_paths'],
|
|
strict_paths=strict, exclude_paths=exclusions, required=path == str(root))
|
|
except NativeScanError as error:
|
|
if checking_free_space:
|
|
self.latest_free_space[path] = error.report['free_space']
|
|
else:
|
|
scans[path] = error.report
|
|
self.latest_scans = scans
|
|
self.latest = {p: scans[p]['logical_bytes'] for p in paths if p in scans}
|
|
error.report = {**self.report(), 'failed_scan': error.report}
|
|
raise
|
|
previous_gaps = {gap['path']: gap for scan in self.latest_scans.values()
|
|
for gap in scan['permission_gaps']}
|
|
current_gaps = {gap['path']: gap for scan in scans.values() for gap in scan['permission_gaps']}
|
|
self.gap_history.append({'boottime': now, 'new': [current_gaps[p] for p in sorted(current_gaps.keys()
|
|
- previous_gaps.keys())],
|
|
'no_longer_observed': [previous_gaps[p] for p in sorted(previous_gaps.keys()
|
|
- current_gaps.keys())],
|
|
'changed': [{'before': previous_gaps[p], 'after': current_gaps[p]}
|
|
for p in sorted(previous_gaps.keys() & current_gaps.keys())
|
|
if previous_gaps[p] != current_gaps[p]]})
|
|
self.latest_scan_completed = True
|
|
self.latest_scans = scans
|
|
self.latest = {p: scans[p]['logical_bytes'] for p in paths}
|
|
if not self.baseline_scans:
|
|
self.baseline_free_space = dict(self.latest_free_space)
|
|
self.baseline_scans = scans
|
|
self.baseline = dict(self.latest)
|
|
self.growth = sum(max(0, size - self.baseline[p]) for p, size in self.latest.items())
|
|
if self.gap_history[-1]['changed']:
|
|
raise RuntimeError('storage_native_permission_gap_identity_changed')
|
|
if any(scan['free_space']['free_bytes'] < self.policy['min_free_bytes'] for scan in scans.values()):
|
|
raise RuntimeError('storage_native_free_space_stop')
|
|
if self.growth >= self.policy['max_native_growth_bytes']:
|
|
raise RuntimeError('storage_native_growth_stop')
|
|
|
|
def report(self):
|
|
return {'scope': self.b.STORAGE_SCOPE, 'native_hard_cap': False,
|
|
'coverage_complete': False, 'runtime_uid': os.geteuid(),
|
|
'latest_scan_completed': self.latest_scan_completed,
|
|
'baseline_skipped_count': sum(s['skipped_count'] for s in self.baseline_scans.values()),
|
|
'latest_skipped_count': sum(s['skipped_count'] for s in self.latest_scans.values()),
|
|
'baseline_logical_bytes': self.baseline, 'latest_logical_bytes': self.latest,
|
|
'observed_positive_growth_bytes': self.growth if self.latest_scan_completed else None,
|
|
'baseline_scans': self.baseline_scans, 'latest_scans': self.latest_scans,
|
|
'baseline_free_space': self.baseline_free_space, 'latest_free_space': self.latest_free_space,
|
|
'permission_gap_history': list(self.gap_history), 'policy': self.policy}
|
|
|
|
|
|
class ForwardingEvidence:
|
|
"""Stable runtime callbacks, installed before constructor; targets fail closed."""
|
|
recorder = None
|
|
observer = None
|
|
|
|
def __call__(self, event, state):
|
|
if self.recorder is None:
|
|
raise RuntimeError('recording_forwarder_uninstalled')
|
|
return self.recorder(event, state)
|
|
|
|
def seam(self, event, state):
|
|
if self.recorder is None:
|
|
raise RuntimeError('recording_forwarder_uninstalled')
|
|
return self.recorder.seam_callback(event, state)
|
|
|
|
def supplemental(self, state):
|
|
if self.observer is None:
|
|
raise RuntimeError('recording_observer_uninstalled')
|
|
return self.observer.supplemental(state)
|
|
|
|
|
|
def bounded_traceback(error):
|
|
"""Retain the innermost 32 frames, without locals, source lines or values."""
|
|
frames = []
|
|
omitted = 0
|
|
tb = error.__traceback__
|
|
while tb is not None:
|
|
code = tb.tb_frame.f_code
|
|
frames.append({'file': code.co_filename[-512:], 'line': tb.tb_lineno,
|
|
'function': code.co_name[:128]})
|
|
if len(frames) > 32:
|
|
del frames[0]
|
|
omitted += 1
|
|
tb = tb.tb_next
|
|
return {'frames': frames, 'omitted_outer_frames': omitted}
|
|
|
|
|
|
def external_failure(phase, error):
|
|
prefix = {'inference': 'inference', 'identity': 'identity', 'recording': 'recording',
|
|
'finalize': 'recording_persistence'}.get(phase, 'infrastructure')
|
|
if isinstance(error, (KeyboardInterrupt, InterruptedError)):
|
|
prefix = 'process_signal'
|
|
context = {'phase': phase, 'type': type(error).__name__, 'message': str(error),
|
|
'traceback': bounded_traceback(error)}
|
|
diagnostic = getattr(error, 'binding_diagnostic', None)
|
|
if diagnostic is not None:
|
|
context['binding_diagnostic'] = diagnostic
|
|
return {'causes': [prefix + '_external_failure'], 'fatal': True, 'context': context}
|
|
|
|
|
|
def execute_lifecycle(api, cfg, protocol, indices, output, budget, verify):
|
|
"""One constructor, stable forwards, actual step results, durable P2 transitions.
|
|
|
|
api is an explicit CPU-test seam; CLI always obtains it from real_runtime.
|
|
There is no plugin/provider flag or production verifier injection path.
|
|
"""
|
|
rows, failures = [], []
|
|
continuation_allowed = True
|
|
env = hooks = recorder = None
|
|
forward = ForwardingEvidence()
|
|
current = indices[0]
|
|
phase = 'identity'
|
|
counts = None
|
|
try:
|
|
verify(current)
|
|
budget.check()
|
|
with api.hooks(before_step=budget.before_step) as hooks:
|
|
try:
|
|
for position, current in enumerate(indices):
|
|
cfg.seed = protocol.trial(current).seed # sole declared transition
|
|
phase = 'identity'
|
|
verify(current) # same used cfg, never normalize here
|
|
budget.check()
|
|
phase = 'recording'
|
|
recorder = api.recorder(protocol, current, output / f'trial-{current:02d}',
|
|
budget_check=budget.recording, supplemental=forward.supplemental)
|
|
forward.recorder = recorder
|
|
stop = failure = None
|
|
acknowledged = False
|
|
report = None
|
|
try:
|
|
phase = 'initialization'
|
|
if position == 0:
|
|
env = api.environment(cfg=cfg, protocol=protocol, trial_index=current,
|
|
runtime_hooks=hooks, evidence_callback=forward,
|
|
seam_callback=forward.seam)
|
|
phase = 'identity'
|
|
verify(current) # immediate constructor boundary, before reset/action
|
|
phase = 'initialization'
|
|
# Enter only after construction, before any published reset snapshot.
|
|
observers = ExitStack()
|
|
try:
|
|
forward.observer = observers.enter_context(api.instrumentation(
|
|
env, backend_rng=api.backend_rng, generators={}))
|
|
except BaseException:
|
|
observers.close()
|
|
raise
|
|
observation, _ = env.reset()
|
|
else:
|
|
observation, _ = env.begin_trial(protocol, current)
|
|
stop = env.stop_result
|
|
if protocol.document['stage'] == 'P2' and current % 2:
|
|
phase = 'identity'
|
|
initial = recorder.audit.report().get('initial_pair_state')
|
|
if (not rows or rows[-1]['trial']['index'] != current - 1
|
|
or not isinstance(initial, dict) or not initial
|
|
or rows[-1].get('initial_pair_state') != initial):
|
|
raise RuntimeError('identity_pair_initial_state_mismatch')
|
|
tick = 0
|
|
while stop is None:
|
|
budget.check()
|
|
phase = 'inference'
|
|
x = observation['policy'] # raw 46D evidence, not normalized model features
|
|
action, mean, std = api.infer(x, protocol.trial(current).controller_kind)
|
|
phase = 'recording'
|
|
causes = recorder.before_control(tick, x, action, mean=mean, std=std)
|
|
if causes:
|
|
raise RuntimeError('Rejected control evidence: ' + repr(causes))
|
|
phase = 'step'
|
|
actual = env.step(action)
|
|
stop = actual.stop
|
|
tick += actual.ticks_advanced
|
|
phase = 'recording'
|
|
causes = recorder.after_step(actual)
|
|
if causes:
|
|
raise RuntimeError('Rejected actual step evidence: ' + repr(causes))
|
|
observation = actual.observation
|
|
except BaseException as error:
|
|
failure = external_failure(phase, error)
|
|
failures.append(failure)
|
|
if env is not None:
|
|
stop = env.stop_result # never manufacture StopResult/actions/ticks
|
|
finally:
|
|
def acknowledge(exact):
|
|
nonlocal acknowledged
|
|
env.acknowledge_finalized(exact)
|
|
acknowledged = True
|
|
|
|
try:
|
|
phase = 'finalize'
|
|
report = recorder.finalize(stop=stop, external_failure=failure,
|
|
acknowledge=acknowledge if failure is None else None)
|
|
except BaseException as error:
|
|
failures.append(external_failure('finalize', error))
|
|
report = recorder.audit.report()
|
|
report.update(status='persistence_failed', functional_passed=False)
|
|
# Preserve partial stream; close it without claiming durability.
|
|
try:
|
|
recorder.stream.close()
|
|
except BaseException as close_error:
|
|
failures.append(external_failure('finalize', close_error))
|
|
rows.append(report)
|
|
if failure is not None or failures or stop is None or stop.fatal or not acknowledged:
|
|
continuation_allowed = False
|
|
break
|
|
recorder = None
|
|
finally:
|
|
phase = 'cleanup'
|
|
try:
|
|
if 'observers' in locals():
|
|
observers.close()
|
|
finally:
|
|
if env is not None:
|
|
env.close() # hooks still installed, cleanup steps cannot evade counting
|
|
except BaseException as error:
|
|
failures.append(external_failure(phase, error))
|
|
finally:
|
|
if hooks is not None:
|
|
counts = {name: getattr(hooks, name) for name in (
|
|
'attempted_steps', 'completed_steps', 'initialization_steps', 'rollout_steps')}
|
|
try:
|
|
verify(current) # used actual cfg, after close/guard unwind, including construction failure
|
|
except BaseException as error:
|
|
failures.append(external_failure('identity', error))
|
|
matrix = api.matrix(protocol, rows)
|
|
matrix.update(failures=failures, observed_counts=counts, selected_rows=indices, last_trial_index=current,
|
|
equality_scope='recorded_application_controller_and_enumerated_observable_rng_only',
|
|
complete_backend_state_equality=False, deterministic_replay=False,
|
|
continuation_eligible=continuation_allowed and not failures and all(
|
|
row.get('terminal', {}).get('runtime_stop') is not None
|
|
and not row['terminal']['runtime_stop']['fatal'] for row in rows)
|
|
and len(rows) == len(indices), accepted_skill=False, optimizer_steps=0)
|
|
return matrix
|
|
|
|
|
|
def preimport_replicator():
|
|
"""Only the intended optional package may be absent; broken dependencies fail."""
|
|
missing = None
|
|
try:
|
|
importlib.import_module(OPTIONAL_REPLICATOR)
|
|
except ModuleNotFoundError as error:
|
|
if error.name not in ('omni.replicator', OPTIONAL_REPLICATOR):
|
|
raise
|
|
missing = error.name
|
|
return {'module': OPTIONAL_REPLICATOR, 'available': missing is None,
|
|
'missing_module': missing, 'sampling_scope': 'no configured sampling',
|
|
'scope_requires_nominal_config_qualification': True,
|
|
'complete_rng_state_claim': False}
|
|
|
|
|
|
def check_replicator_inventory(inventory):
|
|
"""Availability is bound by exact reviewed module membership, not a grant."""
|
|
if (OPTIONAL_REPLICATOR in sys.modules) != (OPTIONAL_REPLICATOR in inventory['modules']):
|
|
raise RuntimeError('identity_optional_replicator_availability_changed')
|
|
|
|
|
|
def stored_config_fields(value):
|
|
"""Binding's dataclass/ClassVar/instance/slot field set, without properties."""
|
|
missing = object()
|
|
|
|
def stored(name):
|
|
item = inspect.getattr_static(value, name, missing)
|
|
if item is missing:
|
|
raise ValueError('Unstored dataclass config field: ' + name)
|
|
if type(item) is MemberDescriptorType:
|
|
return item.__get__(value, type(value))
|
|
if inspect.getattr_static(type(item), '__get__', None) is not None:
|
|
# Functions/classes are legitimate stored values, not evaluated descriptors.
|
|
if not (inspect.isfunction(item) or inspect.isclass(item) or inspect.isbuiltin(item)):
|
|
raise ValueError('Unsupported config field descriptor: ' + name)
|
|
return item
|
|
|
|
fields = {name: stored(name) for name in inspect.getattr_static(type(value), '__dataclass_fields__')}
|
|
dictionary = next((vars(cls)['__dict__'] for cls in type(value).__mro__ if '__dict__' in vars(cls)), None)
|
|
if type(dictionary) is GetSetDescriptorType:
|
|
fields.update(dictionary.__get__(value, type(value)))
|
|
elif dictionary is not None:
|
|
raise ValueError('Unsupported config instance dictionary descriptor')
|
|
for cls in type(value).__mro__:
|
|
slots = vars(cls).get('__slots__', ())
|
|
for slot in (slots,) if type(slots) is str else slots:
|
|
if slot not in ('__dict__', '__weakref__') and inspect.getattr_static(value, slot, missing) is not missing:
|
|
with suppress(AttributeError): # uninitialized slot, as in binding._config_fields
|
|
fields[slot] = stored(slot)
|
|
return fields
|
|
|
|
|
|
def config_references(config, binding):
|
|
"""Collect exact ResolvableStrings, rejecting invalid graphs before imports."""
|
|
references, active = [], set()
|
|
|
|
def visit(value):
|
|
if binding._resolvable(value):
|
|
reference = str(value)
|
|
if reference.count(':') != 1:
|
|
raise ValueError('Unsupported ResolvableString reference')
|
|
module, attribute = reference.split(':')
|
|
if not attribute.isidentifier() or not all(p.isidentifier() for p in module.split('.')):
|
|
raise ValueError('Unsupported ResolvableString reference')
|
|
references.append(value)
|
|
return
|
|
if value is None or type(value) in (str, bool, int, Path, PosixPath, WindowsPath):
|
|
return
|
|
if type(value) is float:
|
|
if not math.isfinite(value):
|
|
raise ValueError('Nonfinite effective config')
|
|
return
|
|
if inspect.isclass(value) or inspect.isfunction(value) or inspect.isbuiltin(value):
|
|
return
|
|
if id(value) in active:
|
|
raise ValueError('Cyclic effective config')
|
|
active.add(id(value))
|
|
try:
|
|
if isinstance(value, enum.Enum):
|
|
children = (value.value,)
|
|
elif type(value) in (dict, MappingProxyType):
|
|
if any(type(k) is not str for k in value):
|
|
raise ValueError('Effective config mapping keys must be strings')
|
|
children = value.values()
|
|
elif type(value) in (tuple, list):
|
|
children = value
|
|
elif type(value) is slice:
|
|
children = (value.start, value.stop, value.step)
|
|
elif type(value) is functools.partial:
|
|
children = (value.func, value.args, value.keywords)
|
|
elif dataclasses.is_dataclass(type(value)):
|
|
children = stored_config_fields(value).values()
|
|
else:
|
|
raise ValueError('Unsupported effective config type: ' + type(value).__qualname__)
|
|
for child in children:
|
|
visit(child)
|
|
finally:
|
|
active.remove(id(value))
|
|
|
|
visit(config)
|
|
return references
|
|
|
|
|
|
def config_module_path(name, binding):
|
|
"""Only exact file-backed references in the existing bound source namespaces.
|
|
|
|
No find_spec parent imports, prefix wildcard imports, or external inventory
|
|
inference. Loaded parent search paths must also be the installed directories.
|
|
"""
|
|
roots = {'isaaclab': binding.ISAAC_ROOT / 'source/isaaclab/isaaclab',
|
|
'isaaclab_physx': binding.ISAAC_ROOT / 'source/isaaclab_physx/isaaclab_physx',
|
|
'SkillFactory': ROOT / binding.PACKAGE}
|
|
parts = name.split('.')
|
|
if parts[0] not in roots:
|
|
raise ValueError('Config reference requires explicit external inventory: ' + name)
|
|
directory = roots[parts[0]]
|
|
for index in range(len(parts)):
|
|
if index:
|
|
directory = directory / parts[index]
|
|
package = directory / '__init__.py'
|
|
path = package if package.is_file() else directory.with_suffix('.py')
|
|
if not path.is_file():
|
|
raise ModuleNotFoundError('Missing installed config module: ' + name, name=name)
|
|
path = binding._file(path)
|
|
parent_name = '.'.join(parts[:index + 1])
|
|
module = sys.modules.get(parent_name)
|
|
if module is not None:
|
|
if binding._module_file(parent_name) != path:
|
|
raise ValueError('Config reference installed source mismatch: ' + parent_name)
|
|
if path == package and list(vars(module).get('__path__', ())) != [str(directory)]:
|
|
raise ValueError('Config reference package search path mismatch: ' + parent_name)
|
|
if index == 0 and module is None:
|
|
raise ValueError('Config reference requires already imported installed root: ' + parent_name)
|
|
if index < len(parts) - 1 and path != package:
|
|
raise ValueError('Config reference parent is not an installed package: ' + parent_name)
|
|
return path
|
|
|
|
|
|
def preimport_config_references(config, binding):
|
|
"""Explicit runner import phase; never calls references or changes their caches.
|
|
|
|
The unchanged encoder/import inventory binds these modules and their source /
|
|
direct dependency closure later. This report is discovery, not a trust grant.
|
|
"""
|
|
references = config_references(config, binding)
|
|
names = sorted({str(value).split(':')[0] for value in references})
|
|
paths = {name: config_module_path(name, binding) for name in names}
|
|
report = {'references': sorted({str(value) for value in references}), 'modules': {}}
|
|
for name, expected in paths.items():
|
|
importlib.import_module(name) # absence and transitive failures deliberately propagate
|
|
path = binding._module_file(name)
|
|
if path != expected or config_module_path(name, binding) != expected:
|
|
raise ValueError('Config reference imported source mismatch: ' + name)
|
|
report['modules'][name] = {'path': str(path), 'sha256': binding.sha256(path.read_bytes())}
|
|
for value in references:
|
|
binding._reference_target(value, set()) # static named-symbol/cache validation only
|
|
return report
|
|
|
|
|
|
def preforward_video_recorder(cfg, env_module):
|
|
"""Mirror pinned ManagerBasedRLEnv.__init__ forwarding before config binding."""
|
|
env_module.verify_installed_stepping(Path('/home/cen/IsaacLab/source/isaaclab/isaaclab/envs'))
|
|
# EvaluationContactCycleEnv calls super().__init__(cfg), i.e. render_mode=None.
|
|
if cfg.video_recorder is not None:
|
|
cfg.video_recorder.env_render_mode = None
|
|
cfg.video_recorder.eye = tuple(float(x) for x in cfg.viewer.eye)
|
|
cfg.video_recorder.lookat = tuple(float(x) for x in cfg.viewer.lookat)
|
|
|
|
|
|
def preimport_constructor_modules(binding):
|
|
"""Eagerly resolve the finite reviewed constructor set; never hide new imports.
|
|
|
|
Transitive imports remain subject to the unchanged full binding inventory and
|
|
immediate postconstructor/final equality checks, not an import ignorelist.
|
|
"""
|
|
report = {}
|
|
for name, (filename, expected_hash) in CONSTRUCTOR_IMPORT_PINS.items():
|
|
expected = Path(filename)
|
|
if (config_module_path(name, binding) != expected
|
|
or binding.sha256(expected.read_bytes()) != expected_hash):
|
|
raise ValueError('Constructor import source mismatch: ' + name)
|
|
importlib.import_module(name)
|
|
if binding._module_file(name) != expected:
|
|
raise ValueError('Constructor imported source mismatch: ' + name)
|
|
report[name] = {'path': filename, 'sha256': binding.sha256(expected.read_bytes())}
|
|
if report[name]['sha256'] != expected_hash:
|
|
raise ValueError('Constructor import source changed: ' + name)
|
|
return report
|
|
|
|
|
|
# Same-named eager submodule imports bypass this package's lazy function export.
|
|
PHYSX_EXPORT_SOURCES = {
|
|
'/home/cen/IsaacLab/source/isaaclab_physx/isaaclab_physx/cloner/__init__.pyi':
|
|
'811bf1b50a42f0c854f13a8c26a9cf1a00e00120132c633c38b58375908f8dc6',
|
|
'/home/cen/isaacsim-6.0.1/kit/python/lib/python3.12/site-packages/lazy_loader/__init__.py':
|
|
'9017e7e780ffe7742b8cd25882c22c93acb9ac353f3372b8dcc76acf832acf29',
|
|
}
|
|
|
|
|
|
def physx_constructor_export(binding, *, resolve=False, expected=None):
|
|
"""Resolve once before binding; later checks never repair namespace drift."""
|
|
package_name = 'isaaclab_physx.cloner'
|
|
module_name = package_name + '.physx_replicate'
|
|
sources = dict(PHYSX_EXPORT_SOURCES)
|
|
for name in (package_name, module_name):
|
|
filename, digest = CONSTRUCTOR_IMPORT_PINS[name]
|
|
if binding._module_file(name) != Path(filename):
|
|
raise ValueError('identity_physx_export_module_path: ' + name)
|
|
sources[filename] = digest
|
|
for filename, digest in sources.items():
|
|
if binding.sha256(Path(filename).read_bytes()) != digest:
|
|
raise ValueError('identity_physx_export_source: ' + filename)
|
|
package, module = sys.modules[package_name], sys.modules[module_name]
|
|
target = vars(module).get('physx_replicate')
|
|
if (not inspect.isfunction(target) or target.__module__ != module_name
|
|
or target.__qualname__ != 'physx_replicate'):
|
|
raise ValueError('identity_physx_export_function')
|
|
binding._source_code(target, Path(CONSTRUCTOR_IMPORT_PINS[module_name][0]))
|
|
if expected is not None and target is not expected:
|
|
raise ValueError('identity_physx_export_changed_callable')
|
|
if resolve:
|
|
resolver = vars(package).get('__getattr__')
|
|
loader_path = Path(next(p for p in PHYSX_EXPORT_SOURCES if '/lazy_loader/' in p))
|
|
if not inspect.isfunction(resolver) or resolver.__module__ != 'lazy_loader':
|
|
raise ValueError('identity_physx_export_resolver')
|
|
binding._source_code(resolver, loader_path)
|
|
if resolver('physx_replicate') is not target:
|
|
raise ValueError('identity_physx_export_resolution')
|
|
if vars(package).get('physx_replicate') is not target:
|
|
raise ValueError('identity_physx_export_namespace')
|
|
return target, {'package': package_name, 'symbol': 'physx_replicate',
|
|
'module': module_name, 'qualname': target.__qualname__, 'sources': sources}
|
|
|
|
|
|
def verify_constructor_export(runtime):
|
|
_, metadata = physx_constructor_export(runtime.binding, expected=runtime.constructor_export)
|
|
if metadata != runtime.generator_inventory['constructor_export']:
|
|
raise ValueError('identity_physx_export_metadata')
|
|
|
|
|
|
def real_runtime(args, output):
|
|
"""Actual imported runtime; no environment/sim.step here, including preparation."""
|
|
os.chdir(ROOT)
|
|
from isaaclab.app import AppLauncher
|
|
|
|
app_module = sys.modules[AppLauncher.__module__]
|
|
if not str(Path(app_module.__file__).resolve()).startswith('/home/cen/IsaacLab/source/'):
|
|
raise RuntimeError('identity_require_actual_/home/cen/IsaacLab_imports')
|
|
app = AppLauncher(headless=True, device=args.device, fast_shutdown=True,
|
|
kit_args=f'--/log/file={Path(args.native_output) / "kit.log"}').app
|
|
try:
|
|
inventory = (None if getattr(args, 'discover_inventory', False)
|
|
else json.loads(Path(args.inventory).read_bytes()))
|
|
# Execution always uses reviewed eager imports; discovery is preparation-only.
|
|
for name in (() if inventory is None else inventory['modules']):
|
|
importlib.import_module(name)
|
|
modules = {name: importlib.import_module(TASK + name) for name in (
|
|
'evaluation_cfg', 'evaluation_env', 'evaluation_recording',
|
|
'evaluation_instrumentation', 'evaluation_rng_scope')}
|
|
for name in ('evaluation_protocol', 'evaluation_audit', 'evaluation_binding',
|
|
'evaluation_budget', 'evaluation_adoption', 'replay'):
|
|
modules[name] = importlib.import_module(LEG + name)
|
|
ins = modules['evaluation_instrumentation']
|
|
for name in ins.BACKEND_SOURCE_INVENTORY:
|
|
importlib.import_module(name)
|
|
# ManagerBasedEnv.seed treats Replicator as optional. Do not enable an
|
|
# extension, mask transitive defects, or claim its hidden RNG is observed.
|
|
replicator = preimport_replicator()
|
|
print(json.dumps({'optional_replicator': replicator}), flush=True)
|
|
if inventory is not None:
|
|
check_replicator_inventory(inventory)
|
|
# Numerical lazy paths also belong in the reviewed inventory.
|
|
import pytest
|
|
import torch
|
|
|
|
if torch.device(args.device).type == 'cuda':
|
|
torch.cuda.init()
|
|
torch.cuda.get_rng_state(torch.device(args.device))
|
|
ins.verify_backend_sources()
|
|
replay = modules['replay']
|
|
if not (ROOT / replay.CANDIDATE).is_file():
|
|
raise RuntimeError('identity_real_artifact_test_cannot_skip_missing_candidate')
|
|
# Required unchanged CPU test in THIS file-backed, correctly imported process.
|
|
if pytest.main(['-q', '-p', 'no:cacheprovider', REAL_TEST]) != 0:
|
|
raise RuntimeError('identity_real_artifact_CPU_test_failed')
|
|
cfg = modules['evaluation_cfg'].EvaluationContactCycleEnvCfg()
|
|
protocol = modules['evaluation_protocol'].load_protocol(ROOT / args.protocol)
|
|
cfg.sim.device = args.device
|
|
cfg.seed = protocol.trial(args.trial_index).seed
|
|
binding = modules['evaluation_binding']
|
|
for name in binding.REQUIRED_RUNTIME:
|
|
importlib.import_module(name)
|
|
preforward_video_recorder(cfg, modules['evaluation_env'])
|
|
constructor_imports = preimport_constructor_modules(binding)
|
|
constructor_export, export_metadata = physx_constructor_export(binding, resolve=True)
|
|
config_imports = preimport_config_references(cfg, binding)
|
|
print(json.dumps({'config_reference_imports': config_imports}), flush=True)
|
|
binding.normalize_config(cfg) # exactly once, actual config used later
|
|
scope = modules['evaluation_rng_scope']
|
|
inventory_report = scope.qualify_config(cfg)
|
|
inventory_report['optional_replicator'] = replicator
|
|
inventory_report['config_reference_imports'] = config_imports
|
|
inventory_report['constructor_imports'] = constructor_imports
|
|
inventory_report['constructor_export'] = export_metadata
|
|
physx_constructor_export(binding, expected=constructor_export)
|
|
ins.qualify_backend_rng(scope.backend_rng(None))
|
|
arguments = dict(root=ROOT, protocol_path=args.protocol, effective_config=cfg,
|
|
future_sources=FUTURE_SOURCES, declared_overrides={'seed': 'protocol.trial.seed'},
|
|
dependency_inventory=inventory, entrypoints=(SCRIPT,))
|
|
api = SimpleNamespace(
|
|
environment=modules['evaluation_env'].EvaluationContactCycleEnv,
|
|
hooks=modules['evaluation_env'].RuntimeHooks.isaac,
|
|
recorder=modules['evaluation_recording'].EvaluationRecorder,
|
|
instrumentation=ins.EvaluationInstrumentation, backend_rng=scope.backend_rng,
|
|
matrix=modules['evaluation_audit'].matrix_report,
|
|
)
|
|
return SimpleNamespace(app=app, cfg=cfg, protocol=protocol, binding=binding,
|
|
arguments=arguments, api=api, replay=replay, torch=torch,
|
|
generator_inventory=inventory_report, constructor_export=constructor_export)
|
|
except BaseException:
|
|
import traceback
|
|
|
|
traceback.print_exc()
|
|
app.close(exit_code=1) # supported fast shutdown must preserve failure
|
|
raise
|
|
|
|
|
|
def runtime_metadata(runtime):
|
|
torch = runtime.torch
|
|
device = torch.device(runtime.cfg.sim.device)
|
|
driver = Path('/proc/driver/nvidia/version')
|
|
return {'torch': str(torch.__version__), 'numpy': str(importlib.import_module('numpy').__version__),
|
|
'warp': str(importlib.import_module('warp').__version__), 'device': str(device),
|
|
'device_name': torch.cuda.get_device_name(device) if device.type == 'cuda' else 'CPU',
|
|
'torch_cuda': torch.version.cuda,
|
|
'driver': driver.read_text() if driver.exists() else 'unavailable',
|
|
'generator_inventory': runtime.generator_inventory,
|
|
'evidence_scope': importlib.import_module(TASK + 'evaluation_rng_scope').EVIDENCE_SCOPE,
|
|
'unobserved_internals': importlib.import_module(TASK + 'evaluation_rng_scope').UNOBSERVED_INTERNALS}
|
|
|
|
|
|
class InventoryPreparationError(ValueError):
|
|
"""Unreviewed diagnostic only; partial identities cannot become a manifest."""
|
|
|
|
def __init__(self, diagnostics):
|
|
self.report = {'status': 'FAIL', 'inventory_status': 'unreviewed_diagnostic',
|
|
'execution_authorized': False, 'environment_constructed': False,
|
|
'sim_step_calls': 0, 'invalid_entries': diagnostics,
|
|
'note': 'Invalid entries were not hashed or accepted; discovery is incomplete.'}
|
|
super().__init__('Inventory preparation FAIL: ' + '; '.join(
|
|
row['entry'] + ': ' + row['error'] for row in diagnostics))
|
|
|
|
|
|
def proposed_native_layouts(roots):
|
|
"""Independent known layouts only for roots with validated file identities."""
|
|
layouts = {}
|
|
if 'torch' in roots:
|
|
layouts['torch'] = (roots['torch'] / 'lib',)
|
|
if 'numpy' in roots:
|
|
layouts['numpy'] = (roots['numpy'] / ('_core' if (roots['numpy'] / '_core').is_dir() else 'core'),
|
|
roots['numpy'] / 'random', roots['numpy'] / 'linalg', roots['numpy'] / 'fft',
|
|
roots['numpy'].parent / 'numpy.libs')
|
|
if 'warp' in roots:
|
|
layouts['warp'] = (roots['warp'] / 'bin',)
|
|
return layouts
|
|
|
|
|
|
def proposed_inventory(binding, cfg):
|
|
"""Collect independent incompatibilities, never approve a partial inventory.
|
|
|
|
Inspect only valid imported identities. An invalid provider is not traversed;
|
|
other independent source/module/native entries can still be diagnosed.
|
|
"""
|
|
diagnostics = []
|
|
|
|
def invalid(stage, entry, error):
|
|
diagnostics.append({'stage': stage, 'entry': entry, 'error': str(error),
|
|
'module_observation': binding.module_diagnostic(entry)})
|
|
|
|
_, names = binding._sources(ROOT, FUTURE_SOURCES, (SCRIPT,))
|
|
roles = set(names)
|
|
binding._encode(cfg, names, set())
|
|
source_records = binding._imports(names, ROOT, diagnostics=diagnostics)
|
|
sources = set(source_records)
|
|
numerical = binding.NUMERICAL_ROOTS
|
|
required = binding.required_dependencies(sources, roles)
|
|
modules = {}
|
|
for name in sorted(required):
|
|
try:
|
|
# Required modules must already be imported by the actual runtime.
|
|
# Never interpret incidental __module__ metadata as an import name.
|
|
modules[name] = binding._dependency_module_identity(name)
|
|
except (ValueError, OSError, ImportError) as error:
|
|
invalid('dependency-module', name, error)
|
|
roots = {name: Path(modules[name]['path']).parent for name in numerical if name in modules}
|
|
layouts = proposed_native_layouts(roots)
|
|
artifacts = {}
|
|
|
|
def add(path, owner, vendor=False):
|
|
try:
|
|
path = binding._file(path)
|
|
label = str(path)
|
|
if label in artifacts:
|
|
raise ValueError('Duplicate proposed native artifact: ' + label)
|
|
artifacts[label] = {'module': owner, 'path': label, 'sha256': binding.sha256(path.read_bytes()),
|
|
'purpose': 'Installed numerical native implementation consumed by ' + owner}
|
|
if vendor:
|
|
artifacts[label]['vendor'] = 'NVIDIA'
|
|
except (ValueError, OSError) as error:
|
|
invalid('native-artifact', str(path), error)
|
|
|
|
for owner, directories in layouts.items():
|
|
for directory in directories:
|
|
paths = sorted(directory.glob('*.so*'))
|
|
if not paths:
|
|
invalid('native-layout', str(directory), 'Unsupported/missing numerical native layout')
|
|
for path in paths:
|
|
add(path, owner)
|
|
if 'torch' in roots and (roots['torch'] / 'lib/libtorch_cuda.so').is_file():
|
|
for package in binding.NVIDIA_CUDA_PACKAGES:
|
|
directory = roots['torch'].parent / 'nvidia' / package / 'lib'
|
|
paths = sorted(directory.glob('*.so*'))
|
|
if package in ('cuda_runtime', 'cublas', 'cudnn') and not paths:
|
|
invalid('native-layout', str(directory), 'Unsupported/missing NVIDIA CUDA native layout')
|
|
for path in paths:
|
|
add(path, 'torch', vendor=True)
|
|
if diagnostics:
|
|
raise InventoryPreparationError(diagnostics)
|
|
proposal = {'scope': binding.INVENTORY_SCOPE, 'modules': modules, 'artifacts': artifacts}
|
|
try:
|
|
return binding._dependency_inventory(proposal, sources, roles)
|
|
except (ValueError, OSError, ImportError) as error:
|
|
invalid('inventory-validation', 'proposal', error)
|
|
raise InventoryPreparationError(diagnostics) from error
|
|
|
|
|
|
def bound_filesystem_inputs(manifest):
|
|
"""Extract immutable file identities, not runtime objects, for external rehash."""
|
|
document = json.loads(manifest)
|
|
files, links = {}, {}
|
|
|
|
def visit(value):
|
|
if isinstance(value, dict):
|
|
if {'path', 'target', 'device', 'inode', 'target_sha256'} <= value.keys():
|
|
links[value['path']] = dict(value)
|
|
if isinstance(value.get('path'), str) and isinstance(value.get('sha256'), str):
|
|
path, digest = value['path'], value['sha256']
|
|
if path in files and files[path] != digest:
|
|
raise ValueError('identity_conflicting_file_hash: ' + path)
|
|
files[path] = digest
|
|
for item in value.values():
|
|
visit(item)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
visit(item)
|
|
|
|
visit(document)
|
|
files.update(document.get('runtime_layout_pins', {}))
|
|
# Frozen bundle verification remains the unchanged child's verify_bundle.
|
|
# Independently inventory every immutable prepared01 file for postexit equality.
|
|
frozen = ROOT / 'logs/contact_cycle/sim_residual_training_v1/prepared01'
|
|
for path in frozen.rglob('*'):
|
|
if path.is_file():
|
|
files[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
return {'files': files, 'links': links, 'frozen_root': str(frozen),
|
|
'scope': 'immutable-files-only; no postexit same-object cfg claim'}
|
|
|
|
|
|
def verify_filesystem_inputs(evidence):
|
|
for filename, expected in evidence.get('links', {}).items():
|
|
path = Path(filename)
|
|
stat = path.lstat()
|
|
target = str(path.readlink())
|
|
if (target != expected['target'] or stat.st_dev != expected['device']
|
|
or stat.st_ino != expected['inode']
|
|
or hashlib.sha256(target.encode()).hexdigest() != expected['target_sha256']):
|
|
raise ValueError('identity_postexit_link_changed: ' + filename)
|
|
for filename, expected in evidence['files'].items():
|
|
path = Path(filename)
|
|
if (not path.is_absolute() or path != path.resolve(strict=True) or path.is_symlink()
|
|
or hashlib.sha256(path.read_bytes()).hexdigest() != expected):
|
|
raise ValueError('identity_postexit_file_changed: ' + filename)
|
|
frozen = Path(evidence['frozen_root'])
|
|
expected = {p for p in evidence['files'] if Path(p).is_relative_to(frozen)}
|
|
if expected != {str(p) for p in frozen.rglob('*') if p.is_file()}:
|
|
raise ValueError('identity_postexit_frozen_file_set_changed')
|
|
return {'status': 'PASS', 'files_rehashed': len(evidence['files']), 'scope': evidence['scope']}
|
|
|
|
|
|
def prepare(args):
|
|
discovery = getattr(args, 'discover_inventory', False)
|
|
if (args.output is None or args.artifact_root is None
|
|
or (args.inventory is None and not discovery) or (args.inventory is not None and discovery)):
|
|
raise ValueError('Preparation requires root/output and exactly one of reviewed inventory or discovery')
|
|
protect_artifact_path(args.output)
|
|
b = cpu_module('evaluation_budget')
|
|
protect_artifact_path(args.artifact_root)
|
|
if not args.output.is_relative_to(args.artifact_root) or args.output == args.artifact_root:
|
|
raise ValueError('Preparation outputs must belong to aggregate artifact root')
|
|
b.durable_directory(args.output)
|
|
native = args.artifact_root / 'native'
|
|
if native != native.resolve():
|
|
raise RuntimeError('storage_native_redirect_symlink')
|
|
if not native.exists():
|
|
b.durable_directory(native)
|
|
native = native / args.output.name
|
|
b.durable_directory(native)
|
|
args.native_output = str(native)
|
|
|
|
def write(path, data):
|
|
if (b.artifact_size(args.artifact_root, exclude_native=True) + len(data)
|
|
> b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE):
|
|
raise RuntimeError('storage_aggregate_cap')
|
|
b.durable_write(path, data)
|
|
|
|
runtime = None
|
|
completed = False
|
|
started = b.boottime()
|
|
try:
|
|
runtime = real_runtime(args, args.output)
|
|
if discovery:
|
|
try:
|
|
inventory = proposed_inventory(runtime.binding, runtime.cfg)
|
|
except InventoryPreparationError as error:
|
|
error.report['module_observations'] = {
|
|
name: runtime.binding.module_diagnostic(name) for name, module in tuple(sys.modules.items())
|
|
if isinstance(module, ModuleType)}
|
|
write(args.output / 'inventory.diagnostic.json', b.encode(error.report))
|
|
raise
|
|
write(args.output / 'inventory.proposed.json', b.encode(inventory))
|
|
runtime.arguments['dependency_inventory'] = inventory
|
|
manifest = runtime.binding.prepare_binding(trial_index=args.trial_index, **runtime.arguments)
|
|
write(args.output / 'binding.json', manifest)
|
|
write(args.output / 'incidental-observations.json', b.encode(
|
|
runtime.binding.incidental_observations(
|
|
set(json.loads(manifest)['imports']) | set(runtime.arguments['dependency_inventory']['modules']))))
|
|
metadata = runtime_metadata(runtime)
|
|
verify_constructor_export(runtime)
|
|
runtime.binding.verify_binding(manifest, trusted_manifest_sha256=hashlib.sha256(manifest).hexdigest(),
|
|
trial_index=args.trial_index, **runtime.arguments)
|
|
filesystem = bound_filesystem_inputs(manifest)
|
|
verify_filesystem_inputs(filesystem)
|
|
write(args.output / 'preclose-filesystem.json', b.encode(filesystem))
|
|
write(args.output / 'preparation.json', b.encode({
|
|
'storage_scope': b.STORAGE_SCOPE, 'native_hard_cap': False, 'launch_identity': launch_identity(),
|
|
'manifest_sha256_for_independent_review': hashlib.sha256(manifest).hexdigest(),
|
|
'execution_authorized': False, 'environment_constructed': False, 'sim_step_calls': 0,
|
|
'inventory_status': 'unreviewed_proposal' if discovery else 'caller_supplied',
|
|
'binding_format': runtime.binding.FORMAT, 'evidence_scope': runtime.binding.PROVENANCE_SCOPE,
|
|
'limitations': runtime.binding.PROVENANCE_LIMITATIONS, 'accepted_skill': False,
|
|
'prepare_seconds': b.boottime() - started, 'runtime': metadata,
|
|
'identity_order': ('actual cfg and inputs verified; evidence durable; then app.close; '
|
|
'parent postexit files/cleanup'),
|
|
'same_object_cfg_scope': 'preclose only; no claim after interpreter destruction',
|
|
'preclose_identity_verified': True, 'shutdown': 'supported-default-fast',
|
|
'final_pass_owner': 'external supervisor after exit',
|
|
'note': 'Preparation timing is not an actual backend rollout/storage qualification.'}))
|
|
completed = True
|
|
except BaseException as error:
|
|
write(args.output / 'preparation-failure.json', b.encode({
|
|
'status': 'FAIL', 'error': repr(error), 'failure': external_failure('preparation', error)}))
|
|
import traceback
|
|
|
|
traceback.print_exc()
|
|
raise
|
|
finally:
|
|
if runtime is not None:
|
|
runtime.app.close(exit_code=0 if completed else 1)
|
|
return 0
|
|
|
|
|
|
def ancestor(parent, child):
|
|
for _ in range(32):
|
|
fields = Path(f'/proc/{child}/stat').read_text().rsplit(') ', 1)[1].split()
|
|
child = int(fields[1])
|
|
if child == parent:
|
|
return True
|
|
if child <= 1:
|
|
return False
|
|
return False
|
|
|
|
|
|
def worker_payload(fd):
|
|
"""Single-use inherited socket; only a live matching supervisor may hand off."""
|
|
with socket.socket(fileno=fd) as channel:
|
|
channel.settimeout(15)
|
|
pid, uid, _ = struct.unpack('3i', channel.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
|
|
command = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0')
|
|
if (Path(f'/proc/{pid}/exe').resolve() != Path('/usr/bin/python3.12')
|
|
or not ancestor(pid, os.getpid())) or uid != os.getuid() or b'--run' not in command or not any(
|
|
item.decode(errors='replace').endswith(SCRIPT) for item in command):
|
|
raise RuntimeError('identity_inherited_parent_supervisor_required')
|
|
channel.sendall(str(os.getpid()).encode() + b'\n')
|
|
with channel.makefile('rb') as stream:
|
|
data = stream.read(1 << 20)
|
|
if len(data) >= 1 << 20:
|
|
raise RuntimeError('identity_oversized_handoff')
|
|
return json.loads(data)
|
|
|
|
|
|
def run_worker(payload):
|
|
b = cpu_module('evaluation_budget')
|
|
if payload.get('launch_identity') != launch_identity():
|
|
raise RuntimeError('identity_worker_launch_changed')
|
|
args = SimpleNamespace(**payload['arguments'])
|
|
for name in ('artifact_root', 'inventory', 'manifest'):
|
|
setattr(args, name, Path(getattr(args, name)))
|
|
reservation = payload['reservation']
|
|
index = reservation['index']
|
|
output = args.artifact_root / f'launch-{index}'
|
|
args.trial_index = reservation['grant']['trial_indices'][0]
|
|
budget = b.LaunchBudget(args.artifact_root, reservation)
|
|
ledger = json.loads((args.artifact_root / 'budget.json').read_bytes())
|
|
if ledger['launches'][-1] != reservation or reservation['status'] != 'reserved':
|
|
raise RuntimeError('identity_stale_reservation')
|
|
budget.write(output / 'worker-claim.json', {'pid': os.getpid(), 'reservation': reservation})
|
|
handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True))
|
|
for s in (signal.SIGTERM, signal.SIGINT)}
|
|
runtime = None
|
|
result = {'accepted_skill': False, 'optimizer_steps': 0, 'continuation_eligible': False,
|
|
'storage_scope': b.STORAGE_SCOPE,
|
|
'observed_counts': None, 'failures': [],
|
|
'rows': [{'trial_index': i, 'status': 'not_run'}
|
|
for i in range(20 if protocol_stage(args.protocol) == 'P2' else 2)]}
|
|
verify = None
|
|
try:
|
|
budget.check()
|
|
manifest = read_trusted(args.manifest, args.trusted_manifest_sha256)
|
|
if b.digest(args.inventory) != reservation['grant']['inventory_sha256']:
|
|
raise RuntimeError('identity_inventory_changed')
|
|
runtime = real_runtime(args, output)
|
|
|
|
def verify(row):
|
|
budget.check(cleanup=True)
|
|
verify_constructor_export(runtime)
|
|
check_replicator_inventory(runtime.arguments['dependency_inventory'])
|
|
return runtime.binding.verify_binding(
|
|
manifest, trusted_manifest_sha256=args.trusted_manifest_sha256,
|
|
trial_index=row, **runtime.arguments)
|
|
|
|
verify(args.trial_index)
|
|
frozen, _ = runtime.replay.verify_bundle(ROOT)
|
|
policy = runtime.replay.load_frozen_policy(ROOT / runtime.replay.CANDIDATE, frozen)
|
|
policy = policy.to(runtime.cfg.sim.device)
|
|
|
|
def infer(x, kind):
|
|
if kind not in ('learned_policy', 'analytic_reference'):
|
|
raise ValueError('identity_unknown_controller')
|
|
mode = 'learned' if kind == 'learned_policy' else 'reference'
|
|
return runtime.replay.inference_action(x, mode, policy if mode == 'learned' else None)
|
|
|
|
runtime.api.infer = infer
|
|
budget.write(output / 'runtime.json', runtime_metadata(runtime))
|
|
result = execute_lifecycle(runtime.api, runtime.cfg, runtime.protocol,
|
|
reservation['grant']['trial_indices'], output, budget, verify)
|
|
except BaseException as error:
|
|
result['failures'].append(external_failure('initialization', error))
|
|
result['continuation_eligible'] = False
|
|
finally:
|
|
if runtime is not None and verify is not None:
|
|
try:
|
|
verify(result.get('last_trial_index', args.trial_index))
|
|
filesystem = bound_filesystem_inputs(manifest)
|
|
verify_filesystem_inputs(filesystem)
|
|
budget.write(output / 'preclose-filesystem.json', filesystem)
|
|
result['preclose_identity_verified'] = True
|
|
except BaseException as error:
|
|
result['failures'].append(external_failure('identity', error))
|
|
result['continuation_eligible'] = False
|
|
result['same_object_cfg_scope'] = 'after env.close, before app.close only'
|
|
result['final_pass_owner'] = 'external supervisor after exit'
|
|
result['storage_scope'] = b.STORAGE_SCOPE
|
|
result['native_hard_cap'] = False
|
|
persisted = False
|
|
try:
|
|
budget.write(output / 'result.json', result)
|
|
persisted = True
|
|
finally:
|
|
for sig, handler in handlers.items():
|
|
signal.signal(sig, handler)
|
|
if runtime is not None:
|
|
runtime.app.close(exit_code=0 if persisted and result['continuation_eligible'] else 1)
|
|
return 0 if result['continuation_eligible'] else 1
|
|
|
|
|
|
def process_table():
|
|
table = {}
|
|
for path in Path('/proc').glob('[0-9]*/stat'):
|
|
try:
|
|
fields = path.read_text().rsplit(') ', 1)[1].split()
|
|
table[int(path.parent.name)] = (int(fields[19]), int(fields[1]), int(fields[2]), fields[0])
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
pass
|
|
return table
|
|
|
|
|
|
class OwnedProcesses:
|
|
"""Dedicated sole-spawner supervisor only; not general service containment.
|
|
|
|
Pre-existing child trees are refused. Track birth identities, not names/PIDs;
|
|
orphan attribution relies on this process spawning no other jobs after start.
|
|
"""
|
|
def __init__(self):
|
|
table = process_table()
|
|
self.supervisor = (os.getpid(), table[os.getpid()][0])
|
|
if any(row[1] == os.getpid() for row in table.values()):
|
|
raise RuntimeError('process_dedicated_supervisor_has_preexisting_children')
|
|
self.preexisting = {(pid, row[0]) for pid, row in table.items()}
|
|
self.known = set()
|
|
self.root = None
|
|
self.unknown = False
|
|
|
|
def attach(self, child):
|
|
row = process_table().get(child.pid)
|
|
if row is None:
|
|
raise RuntimeError('process_spawn_identity_unavailable')
|
|
self.root = (child.pid, row[0])
|
|
self.known.add(self.root)
|
|
|
|
def discover(self):
|
|
table = process_table()
|
|
if table.get(self.supervisor[0], (None,))[0] != self.supervisor[1]:
|
|
self.unknown = True
|
|
return {}
|
|
# Retain exclusions for observed descendants of unrelated baseline
|
|
# processes as well, even if their parent later exits/adopts here.
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for pid, row in table.items():
|
|
identity = (pid, row[0])
|
|
parent = table.get(row[1])
|
|
if (identity not in self.preexisting and identity not in self.known and parent is not None
|
|
and (row[1], parent[0]) != self.supervisor
|
|
and (row[1], parent[0]) in self.preexisting):
|
|
self.preexisting.add(identity)
|
|
changed = True
|
|
if any(row[1] == self.supervisor[0] and (pid, row[0]) in self.preexisting
|
|
for pid, row in table.items()):
|
|
self.unknown = True # unexpected unrelated adoption violates dedicated ownership
|
|
active = {pid: row for pid, row in table.items() if (pid, row[0]) in self.known}
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for pid, row in table.items():
|
|
identity = (pid, row[0])
|
|
if pid in active or identity in self.preexisting:
|
|
continue
|
|
# Direct children after attach are adopted owned orphans under
|
|
# the enforced dedicated, sole-spawner architecture.
|
|
if (self.root is not None or self.unknown) and (row[1] == self.supervisor[0] or row[1] in active):
|
|
self.known.add(identity)
|
|
active[pid] = row
|
|
changed = True
|
|
return active
|
|
|
|
def signal(self, pid, row, sig):
|
|
# pidfd pins the process during signal delivery; recheck starttime after
|
|
# opening to close the discovery -> open PID-reuse window.
|
|
with suppress(ProcessLookupError, FileNotFoundError):
|
|
fd = os.pidfd_open(pid)
|
|
try:
|
|
current = process_table().get(pid)
|
|
if current is not None and current[0] == row[0]:
|
|
signal.pidfd_send_signal(fd, sig)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
def reap(self, child):
|
|
child.poll()
|
|
for pid, row in self.discover().items():
|
|
if pid != child.pid and row[1] == self.supervisor[0] and row[3] == 'Z':
|
|
# An unreaped child cannot reuse its PID. Never reap unrelated jobs.
|
|
with suppress(ChildProcessError, ProcessLookupError):
|
|
os.waitpid(pid, os.WNOHANG)
|
|
|
|
|
|
def cleanup_group(child, deadline, b, owned, pump=lambda: None):
|
|
sent = set()
|
|
term_until = min(deadline - 10, b.boottime() + 110)
|
|
while b.boottime() < deadline:
|
|
sig = signal.SIGTERM if b.boottime() < term_until else signal.SIGKILL
|
|
for pid, row in owned.discover().items():
|
|
key = (pid, row[0], sig)
|
|
if row[3] != 'Z' and key not in sent:
|
|
owned.signal(pid, row, sig)
|
|
sent.add(key)
|
|
owned.reap(child)
|
|
pump() # keep draining even when handshake failed or descendants hold pipe
|
|
if not owned.discover():
|
|
return not owned.unknown
|
|
time.sleep(.05)
|
|
owned.reap(child)
|
|
pump()
|
|
return not owned.discover() and not owned.unknown
|
|
|
|
|
|
class StdoutCapture:
|
|
"""Nonblocking capture from spawn through cleanup; overflow drains/discards.
|
|
|
|
Each of three launches has a fixed 1 MiB allowance withheld from all other
|
|
writers. This avoids a check/write race with the worker's pending trace bytes.
|
|
The 16 MiB cleanup reserve is separate and never consumed by stdout.
|
|
"""
|
|
def __init__(self, pipe, path, b):
|
|
self.pipe, self.b = pipe, b
|
|
self.log = Path(path).open('xb', buffering=0) # noqa: SIM115 - owned until cleanup close
|
|
try:
|
|
os.set_blocking(pipe.fileno(), False)
|
|
except BaseException:
|
|
try:
|
|
self.log.close()
|
|
finally:
|
|
pipe.close()
|
|
raise
|
|
self.written = self.discarded = 0
|
|
self.eof = False
|
|
self.error = None
|
|
|
|
def pump(self):
|
|
# Bounded work per turn, even for an unlimited producer.
|
|
for _ in range(16):
|
|
try:
|
|
block = os.read(self.pipe.fileno(), 65536)
|
|
except BlockingIOError:
|
|
break
|
|
if not block:
|
|
self.eof = True
|
|
break
|
|
retained = block[:max(0, self.b.STDOUT_BYTES - self.written)]
|
|
try:
|
|
if retained and self.error is None:
|
|
if self.log.write(retained) != len(retained):
|
|
raise OSError('storage_short_stdout_write')
|
|
self.written += len(retained)
|
|
except OSError as error:
|
|
self.error = error
|
|
self.discarded += len(block) - (len(retained) if self.error is None else 0)
|
|
return self.discarded
|
|
|
|
def check(self):
|
|
self.pump()
|
|
if self.error is not None:
|
|
raise self.error
|
|
if self.discarded:
|
|
raise RuntimeError('storage_stdout_capture_overflow')
|
|
|
|
def close(self):
|
|
try:
|
|
self.pump()
|
|
os.fsync(self.log.fileno())
|
|
finally:
|
|
self.log.close()
|
|
self.pipe.close()
|
|
|
|
|
|
def exchange_worker(child, channel, payload, budget, capture, owned, monitor, b):
|
|
deadline = min(b.boottime() + 30, budget.reservation['deadline_boottime'] - b.CLEANUP_SECONDS)
|
|
channel.setblocking(False)
|
|
hello = b''
|
|
while b'\n' not in hello:
|
|
capture.check()
|
|
budget.check()
|
|
monitor.check()
|
|
owned.discover()
|
|
if b.boottime() >= deadline:
|
|
raise RuntimeError('identity_worker_handshake_timeout')
|
|
try:
|
|
block = channel.recv(64 - len(hello))
|
|
if not block:
|
|
raise RuntimeError('identity_worker_handshake_eof')
|
|
hello += block
|
|
except BlockingIOError:
|
|
if child.poll() is not None:
|
|
capture.pump()
|
|
raise RuntimeError('identity_worker_exited_without_handshake')
|
|
time.sleep(.01)
|
|
if len(hello) >= 64:
|
|
raise RuntimeError('identity_worker_handshake_oversized')
|
|
worker_pid = int(hello.strip())
|
|
if (os.getpgid(worker_pid) != child.pid or worker_pid not in owned.discover()
|
|
or not ancestor(os.getpid(), worker_pid)):
|
|
raise RuntimeError('identity_foreign_worker_handshake')
|
|
channel.settimeout(max(.01, deadline - b.boottime()))
|
|
channel.sendall(b.encode(payload))
|
|
channel.shutdown(socket.SHUT_WR)
|
|
channel.close()
|
|
|
|
|
|
def supervise(args):
|
|
b = cpu_module('evaluation_budget')
|
|
policy = storage_policy_check(args, b)
|
|
if not hasattr(os, 'pidfd_open') or not hasattr(signal, 'pidfd_send_signal'):
|
|
raise RuntimeError('process_identity_safe_pidfd_supervisor_required')
|
|
grant = grant_check(json.loads(read_trusted(args.grant, args.trusted_grant_sha256)), args, b)
|
|
profile = resource_profile(args.protocol)
|
|
slots, _, _ = b.profile_limits(profile)
|
|
if policy['not_after_boottime'] < max(grant['not_after_boottime'],
|
|
b.boottime() + slots[grant['launch_index']][2]
|
|
+ (b.CLEANUP_SECONDS if profile is not None else 0)):
|
|
raise RuntimeError('storage_policy_expires_before_grant')
|
|
owned = OwnedProcesses() # refuse pre-existing jobs before any reservation
|
|
monitor = NativeMonitor(policy, b)
|
|
environment = dict(os.environ, PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', PYTHONDONTWRITEBYTECODE='1')
|
|
launch_command([], environment) # qualify installed launch path before debit
|
|
ledger = b.BudgetLedger(args.artifact_root, grant['campaign_id'], cap_bytes=policy['limit_bytes'],
|
|
resource_profile=profile)
|
|
child = reservation = channels = capture = None
|
|
clean = eligible = False
|
|
counts = None
|
|
errors = []
|
|
handlers = {}
|
|
monitor_failed = False
|
|
|
|
def cleanup_pump():
|
|
nonlocal monitor_failed
|
|
if capture is not None:
|
|
capture.pump()
|
|
try:
|
|
monitor.check()
|
|
except Exception as error:
|
|
if not monitor_failed:
|
|
errors.append(external_failure('supervisor', error))
|
|
monitor_failed = True # keep signaling/reaping even after native stop
|
|
|
|
try:
|
|
reservation = ledger.reserve(grant, enforcement_deadline=policy['not_after_boottime'])
|
|
output = args.artifact_root / f'launch-{reservation["index"]}'
|
|
budget = b.LaunchBudget(args.artifact_root, reservation)
|
|
b.durable_directory(output)
|
|
handlers = {s: signal.signal(s, lambda *_: setattr(budget, 'interrupted', True))
|
|
for s in (signal.SIGTERM, signal.SIGINT)}
|
|
budget.write(output / 'reservation.json', reservation)
|
|
if profile is not None:
|
|
budget.write(output / 'adopted-p1-evidence.json', cpu_module('evaluation_adoption').verify_adoption(
|
|
args.artifact_root, grant['p1_adoption_sha256']))
|
|
budget.write(output / 'storage-policy.json', monitor.report())
|
|
budget.write(output / 'launch-identity.json', launch_identity())
|
|
budget.write(output / 'supervisor-identity.json', {
|
|
'pid': owned.supervisor[0], 'starttime': owned.supervisor[1],
|
|
'dedicated_sole_spawner': True, 'preexisting_children': []})
|
|
for name, path in (('manifest.json', args.manifest), ('inventory.json', args.inventory),
|
|
('grant.json', args.grant)):
|
|
data = Path(path).read_bytes()
|
|
budget.check(pending=len(data))
|
|
b.durable_write(output / name, data)
|
|
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
|
raise RuntimeError('process_subreaper_setup_failed')
|
|
channels = socket.socketpair()
|
|
parent, child_channel = channels
|
|
child_channel.set_inheritable(True)
|
|
native = args.artifact_root / 'native'
|
|
if native != native.resolve():
|
|
raise RuntimeError('storage_native_redirect_symlink')
|
|
if not native.exists():
|
|
b.durable_directory(native)
|
|
native = native / f'launch-{reservation["index"]}'
|
|
b.durable_directory(native)
|
|
redirects = native_redirects(native)
|
|
for directory in redirects.values():
|
|
b.durable_directory(Path(directory))
|
|
environment.update(redirects)
|
|
command = launch_command([str(ROOT / SCRIPT), '--_worker-fd', str(child_channel.fileno())], environment)
|
|
budget.check()
|
|
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, start_new_session=True,
|
|
pass_fds=(child_channel.fileno(),))
|
|
try:
|
|
owned.attach(child) # first operation after spawn; capture setup can fail
|
|
except BaseException:
|
|
# Sole-spawner discovery must still collect children if birth lookup
|
|
# failed; unknown ownership can never be reported as clean.
|
|
owned.unknown = True
|
|
raise
|
|
child_channel.close()
|
|
capture = StdoutCapture(child.stdout, output / 'stdout.log', b)
|
|
arguments = {'protocol': args.protocol, 'device': args.device,
|
|
'artifact_root': str(args.artifact_root), 'native_output': str(native),
|
|
'inventory': str(output / 'inventory.json'), 'manifest': str(output / 'manifest.json'),
|
|
'trusted_manifest_sha256': args.trusted_manifest_sha256}
|
|
exchange_worker(child, parent, {'arguments': arguments, 'reservation': reservation,
|
|
'launch_identity': launch_identity()},
|
|
budget, capture, owned, monitor, b)
|
|
while not capture.eof or child.poll() is None:
|
|
capture.check()
|
|
budget.check()
|
|
monitor.check()
|
|
owned.discover()
|
|
time.sleep(.01)
|
|
child.wait(timeout=max(.1, reservation['deadline_boottime'] - b.boottime() - 120))
|
|
if (output / 'result.json').is_file():
|
|
result = json.loads((output / 'result.json').read_bytes())
|
|
counts = result['observed_counts']
|
|
postexit = verify_filesystem_inputs(json.loads((output / 'preclose-filesystem.json').read_bytes()))
|
|
budget.write(output / 'postexit-filesystem.json', postexit)
|
|
eligible = (child.returncode == 0 and result['continuation_eligible']
|
|
and result.get('preclose_identity_verified') is True)
|
|
except BaseException as error:
|
|
errors.append(external_failure('supervisor', error))
|
|
finally:
|
|
try:
|
|
try:
|
|
if channels:
|
|
for channel in channels:
|
|
channel.close()
|
|
if child is not None and capture is None and child.stdout is not None:
|
|
child.stdout.close() # constructor/attach failure left no capture owner
|
|
finally:
|
|
if child is not None:
|
|
clean = cleanup_group(child, reservation['deadline_boottime'], b, owned, cleanup_pump)
|
|
if capture is not None:
|
|
capture.close()
|
|
if capture.discarded or capture.error:
|
|
eligible = False
|
|
errors.append(external_failure('recording', RuntimeError('storage_stdout_capture_incomplete')))
|
|
if reservation is not None:
|
|
if profile is not None:
|
|
try:
|
|
adoption = cpu_module('evaluation_adoption').verify_adoption(
|
|
args.artifact_root, grant['p1_adoption_sha256'])
|
|
budget.write(output / 'postexit-adoption.json', {
|
|
'p1_adoption_sha256': grant['p1_adoption_sha256'], 'status': 'PASS',
|
|
'rows': [{'trial': row['trial'], 'first_failure': row['first_failure'],
|
|
'functional_passed': False, 'accepted_skill': False}
|
|
for row in adoption['rows']],
|
|
'historical_reserved_calls': adoption['historical_reserved_calls'], 'local_reserved_calls': 91000,
|
|
'cumulative_reserved_calls': adoption['historical_reserved_calls'] + 91000})
|
|
except BaseException as error:
|
|
eligible = False
|
|
errors.append(external_failure('identity', error))
|
|
final_size = b.artifact_size(args.artifact_root, exclude_native=True)
|
|
eligible = eligible and not errors and final_size < policy['limit_bytes'] - b.HEADROOM_BYTES
|
|
budget.write(output / 'supervisor.json', {
|
|
'cleanup_clean': clean, 'errors': errors, 'returncode': None if child is None else child.returncode,
|
|
'artifact_bytes_before_finalization': final_size, 'continuation_eligible': eligible and clean,
|
|
'storage': monitor.report(), 'stdout_discarded_bytes': capture.discarded if capture else 0,
|
|
'stdout_eof_observed': capture.eof if capture else False,
|
|
'owned_process_identities': sorted(owned.known), 'ownership_unknown': owned.unknown,
|
|
'reserved_physical_calls': reservation['reserved_physical_calls'], 'observed_counts': counts})
|
|
ledger.finish(counts=counts, clean=clean, eligible=eligible)
|
|
finally:
|
|
ledger.close()
|
|
for sig, handler in handlers.items():
|
|
signal.signal(sig, handler)
|
|
return 0 if eligible and clean else 1
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
modes = parser.add_mutually_exclusive_group(required=True)
|
|
modes.add_argument('--check-only', action='store_true')
|
|
modes.add_argument('--prepare-binding', action='store_true')
|
|
modes.add_argument('--run', action='store_true')
|
|
modes.add_argument('--_worker-fd', type=int, help=argparse.SUPPRESS)
|
|
parser.add_argument('--protocol', choices=PROTOCOLS, default=PROTOCOLS[0])
|
|
parser.add_argument('--trial-index', type=int, default=0)
|
|
parser.add_argument('--device', default='cuda:0')
|
|
parser.add_argument('--inventory', type=Path)
|
|
parser.add_argument('--discover-inventory', action='store_true',
|
|
help='Preparation only: propose actual imported inventory; never grants execution trust')
|
|
parser.add_argument('--output', type=Path)
|
|
parser.add_argument('--artifact-root', type=Path)
|
|
parser.add_argument('--manifest', type=Path)
|
|
parser.add_argument('--trusted-manifest-sha256')
|
|
parser.add_argument('--grant', type=Path)
|
|
parser.add_argument('--trusted-grant-sha256')
|
|
parser.add_argument('--storage-policy', type=Path)
|
|
parser.add_argument('--trusted-storage-policy-sha256')
|
|
args = parser.parse_args(argv)
|
|
if args.discover_inventory and (not args.prepare_binding or args.inventory is not None):
|
|
parser.error('--discover-inventory requires preparation only, without --inventory')
|
|
if args._worker_fd is not None:
|
|
return run_worker(worker_payload(args._worker_fd))
|
|
if args.check_only:
|
|
protocol = cpu_module('evaluation_protocol').load_protocol(ROOT / args.protocol)
|
|
protocol.trial(args.trial_index)
|
|
print(json.dumps({'scope': 'CPU protocol/source planning only; no binding, authorization or physical pass',
|
|
'stage': protocol.document['stage'], 'rows': [asdict(t) for t in protocol.trials],
|
|
'future_sources': FUTURE_SOURCES,
|
|
'resource_profile': resource_profile(args.protocol),
|
|
'reserved_physical_calls': [slot[0] for slot in
|
|
cpu_module('evaluation_budget').profile_limits(resource_profile(args.protocol))[0]],
|
|
'actual_backend_imports_and_performance': 'unqualified'}))
|
|
return 0
|
|
if args.prepare_binding:
|
|
return prepare(args)
|
|
if any(getattr(args, name) is None for name in ('inventory', 'artifact_root', 'manifest',
|
|
'trusted_manifest_sha256', 'grant', 'trusted_grant_sha256')):
|
|
parser.error('--run requires reviewed inventory, artifact root, manifest+trusted digest, '
|
|
'fresh grant+trusted digest')
|
|
return supervise(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|