Files
SkillFactory/tests/test_contact_cycle_sim_evaluation_runner.py
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

1622 lines
78 KiB
Python

"""CPU lifecycle doubles plus one explicitly harmless installed FD exec-chain probe. No Kit."""
import importlib.util
import json
import sys
from pathlib import Path
from types import SimpleNamespace as NS
import pytest
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location(
'evaluation_runner_test_cli', ROOT / 'scripts/evaluate_cycle_sim_residual.py')
cli = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = cli
spec.loader.exec_module(cli)
b = cli.cpu_module('evaluation_budget')
protocols = cli.cpu_module('evaluation_protocol')
@pytest.fixture(autouse=True)
def no_real_launch(monkeypatch, request):
if request.node.name == 'test_harmless_installed_launch_probe':
return
def forbidden(*args, **kwargs):
raise AssertionError('No real launcher/process calls in runner CPU tests')
monkeypatch.setattr(cli.subprocess, 'Popen', forbidden)
if not hasattr(cli.os, 'pidfd_open'):
monkeypatch.setattr(cli.os, 'pidfd_open', forbidden, raising=False)
@pytest.fixture
def fake_clock():
value = NS(now=100., boot='test-boot')
value.clock = lambda: value.now
value.boot_fn = lambda: value.boot
return value
def reservation(index=0, clock=100.):
ticks, rollout, seconds = b.SLOTS[index]
return {'index': index, 'status': 'reserved', 'reserved_physical_calls': ticks,
'rollout_cap': rollout, 'initialization_cap': 1000, 'started_boottime': clock,
'deadline_boottime': clock + seconds, 'grant': {'boot_id': 'test-boot'}}
def test_budget_conservative_debits_and_no_resume(tmp_path, fake_clock):
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
grant = {'launch_index': 0, 'not_after_boottime': 1000}
row = ledger.reserve(grant)
assert row['reserved_physical_calls'] == 14500 and row['observed_counts'] is None
with pytest.raises(RuntimeError, match='retry|reused'):
ledger.reserve(grant)
ledger.close() # simulate crash, no completion record
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
with pytest.raises(RuntimeError, match='interrupted'):
ledger.reserve({**grant, 'launch_index': 1})
assert ledger.data['launches'][0]['reserved_physical_calls'] == 14500
ledger.close()
def test_no_overlap_boot_change_or_campaign_substitution(tmp_path, fake_clock):
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
with pytest.raises(BlockingIOError):
b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
ledger.close()
fake_clock.boot = 'different-boot'
with pytest.raises(RuntimeError, match='boot_mismatch'):
b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
def test_three_slots_reviewed_P2_no_refunds_or_fourth(tmp_path, fake_clock):
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
for i in range(3):
if i < 2:
(tmp_path / f'launch-{i}').mkdir()
(tmp_path / f'launch-{i}' / 'result').write_text(str(i))
grant = {'launch_index': i, 'not_after_boottime': 4000,
'p1_evidence_sha256': [b.tree_digest(tmp_path / f'launch-{j}') for j in (0, 1)] if i == 2 else []}
ledger.reserve(grant)
ledger.finish(counts={'completed_steps': 1}, clean=True, eligible=True)
fake_clock.now += 10
assert sum(r['reserved_physical_calls'] for r in ledger.data['launches']) == 120000
assert all(r['observed_counts']['completed_steps'] == 1 for r in ledger.data['launches'])
with pytest.raises(RuntimeError):
ledger.reserve({'launch_index': 3})
ledger.close()
def test_cleanup_and_aggregate_boottime_not_prepare_time(tmp_path, fake_clock):
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
fake_clock.now = 5000 # coding/preparation time is not charged
row = ledger.reserve({'launch_index': 0, 'not_after_boottime': 10000})
assert ledger.data['first_run_boottime'] == 5000
row['grant']['boot_id'] = 'test-boot'
budget = b.LaunchBudget(tmp_path, row, clock=fake_clock.clock, boot=fake_clock.boot_fn)
fake_clock.now = 5480
with pytest.raises(RuntimeError, match='time'):
budget.check()
budget.check(cleanup=True)
ledger.finish(counts=None, clean=True, eligible=True)
fake_clock.now = 8480
with pytest.raises(RuntimeError, match='aggregate'):
ledger.reserve({'launch_index': 1, 'not_after_boottime': 10000})
ledger.close()
@pytest.mark.parametrize('mode,init,roll,attempt,allowed', [
('construction', 999, 0, 1000, True), ('initialization', 1000, 0, 1001, False),
('rollout', 0, 13499, 13500, True), ('rollout', 0, 13500, 13501, False),
('rollout', 999, 13499, 14501, False),
])
def test_all_initialization_reset_and_rollout_caps(tmp_path, fake_clock, mode, init, roll, attempt, allowed):
budget = b.LaunchBudget(tmp_path, reservation(), clock=fake_clock.clock, boot=fake_clock.boot_fn)
h = NS(mode=mode, initialization_steps=init, rollout_steps=roll, attempted_steps=attempt)
if allowed:
budget.before_step(h)
else:
with pytest.raises(RuntimeError, match='physical_calls'):
budget.before_step(h)
def test_exact_pending_artifacts_headroom_signal_and_native_scope(tmp_path, fake_clock, monkeypatch):
monkeypatch.setattr(b, 'CAP_BYTES', 1000)
monkeypatch.setattr(b, 'HEADROOM_BYTES', 100)
monkeypatch.setattr(b, 'STDOUT_ALLOWANCE', 0)
(tmp_path / 'stdout.log').write_bytes(b'x' * 800)
budget = b.LaunchBudget(tmp_path, reservation(), clock=fake_clock.clock, boot=fake_clock.boot_fn)
budget.recording('record', 100, 0)
with pytest.raises(RuntimeError, match='storage'):
budget.recording('record', 101, 0)
budget.recording('finalize', 200, 0)
with pytest.raises(RuntimeError, match='storage'):
budget.recording('finalize', 201, 0)
budget.interrupted = True
with pytest.raises(InterruptedError):
budget.check()
budget.check(cleanup=True)
(tmp_path / 'link').symlink_to(tmp_path / 'stdout.log')
with pytest.raises(RuntimeError, match='nonregular'):
budget.check(cleanup=True)
def test_artifact_scan_restarts_after_atomic_publication_not_partial_total(tmp_path, monkeypatch):
target = tmp_path / 'physics-counts.json'
pending = tmp_path / 'physics-counts.json.pending'
target.write_bytes(b'x')
pending.write_bytes(b'y' * 700)
scans = []
original = Path.lstat
def walk(root, **kwargs):
scans.append(True)
yield str(root), [], [target.name, pending.name] if len(scans) == 1 else [target.name]
def lstat(path, *args, **kwargs):
if path == pending:
pending.replace(target)
raise FileNotFoundError('atomic rename between listing and lstat')
return original(path, *args, **kwargs)
monkeypatch.setattr(b.os, 'walk', walk)
monkeypatch.setattr(Path, 'lstat', lstat)
assert b.artifact_size(tmp_path) == 700
assert len(scans) == 2
def test_artifact_scan_persistent_churn_is_bounded_and_permission_denied_fails(tmp_path, monkeypatch):
scans = []
def churn(root, **kwargs):
scans.append(True)
yield str(root), [], ['always-disappeared.pending']
monkeypatch.setattr(b.os, 'walk', churn)
with pytest.raises(RuntimeError, match='unstable_after_4'):
b.artifact_size(tmp_path)
assert len(scans) == 4
def inaccessible(root, **kwargs):
kwargs['onerror'](PermissionError('unreadable owned directory'))
yield # generator shape; must never yield a partial success
monkeypatch.setattr(b.os, 'walk', inaccessible)
with pytest.raises(PermissionError, match='unreadable'):
b.artifact_size(tmp_path)
@pytest.fixture(params=[1, 2])
def lifecycle(tmp_path, request): # noqa: C901 - scoped CPU doubles keep lifecycle events in one fixture
"""Deterministic environment/recorder boundaries with real runner control flow."""
events = []
protocol = protocols.load_protocol(ROOT / cli.PROTOCOLS[request.param])
cfg = NS(seed=protocol.trial(0).seed, mutation=False)
fault = NS(phase=None, row=0, fatal=False, acknowledge=True, mutate=False)
state = NS(env=None, hooks=None, recorders=[])
class Hooks:
def __init__(self, before_step):
self.before_step = before_step
self.attempted_steps = self.completed_steps = self.initialization_steps = self.rollout_steps = 0
state.hooks = self
def __enter__(self):
events.append('hooks_enter')
return self
def step(self, mode):
self.mode = mode
self.attempted_steps += 1
self.before_step(self)
self.completed_steps += 1
if mode == 'rollout':
self.rollout_steps += 1
else:
self.initialization_steps += 1
def __exit__(self, *_):
events.append('hooks_exit')
class Env:
def __init__(self, **kwargs):
assert events[-1] == 'recorder_0'
state.env = self
self.cfg = kwargs['cfg']
self.index = kwargs['trial_index']
self.evidence = kwargs['evidence_callback']
self.stop_result = None
self.hooks = kwargs['runtime_hooks']
self.acked = False
self.hooks.step('construction')
events.append('construct')
if fault.phase == 'constructor_config':
self.cfg.mutation = True
if fault.phase == 'construct':
raise RuntimeError('constructor fault')
def reset(self):
events.append(f'reset_{self.index}')
self.hooks.step('initialization')
self.evidence('initial', self)
return {'policy': 'raw46'}, {}
def step(self, action):
assert action == 'actual_action'
events.append(f'step_{self.index}')
self.hooks.step('rollout')
self.evidence('tick', self)
self.stop_result = NS(fatal=fault.fatal, causes=('functional_fail',), trial_index=self.index)
return NS(stop=self.stop_result, ticks_advanced=1, observation=None, reward=None)
def acknowledge_finalized(self, stop):
assert stop is self.stop_result
assert events[-1] == f'durable_{self.index}'
self.acked = True
events.append(f'ack_{self.index}')
def begin_trial(self, proto, index):
assert proto is protocol and index == self.index + 1 and self.acked
assert (tmp_path / f'trial-{self.index:02d}' / 'durable.json').exists()
self.index, self.acked, self.stop_result = index, False, None
return self.reset()
def close(self):
if 'observer_enter' in events:
assert events[-1] == 'observer_exit'
events.append('env_close')
if fault.mutate:
self.cfg.mutation = True
class Recorder:
def __init__(self, proto, row, path, *, supplemental, budget_check):
self.row, self.supplemental, self.path = row, supplemental, path
self.audit = NS(report=lambda: self.report())
path.mkdir()
self.stream = (path / 'partial').open('wb')
events.append(f'recorder_{row}')
state.recorders.append(self)
def __call__(self, event, env):
assert self.supplemental(env) == 'snapshot'
events.append(f'evidence_{event}_{self.row}')
return ()
def seam_callback(self, event, env):
return ()
def before_control(self, tick, x, action, **kwargs):
assert tick == 0 and x == 'raw46'
events.append(f'control_{self.row}')
return ('recording_bad_control',) if fault.phase == 'control' else ()
def after_step(self, actual):
assert actual is not None and actual.ticks_advanced == 1 and actual.reward is None
events.append(f'after_step_{self.row}')
return ('recording_bad_step',) if fault.phase == 'after_step' else ()
def report(self):
return {'trial': json.loads(json.dumps(cli.asdict(protocol.trial(self.row)))),
'initial_pair_state': {'mock_initial': self.row if fault.phase == 'pair_identity' else 0},
'functional_passed': False, 'all_causes': ['functional_fail']}
def finalize(self, *, stop, external_failure, acknowledge):
events.append(f'finalize_{self.row}')
if fault.phase == 'finalize':
raise OSError('fsync fault')
b.durable_write(self.path / 'durable.json', b.encode({'external': external_failure}))
self.stream.close()
events.append(f'durable_{self.row}')
if acknowledge and fault.acknowledge and stop is not None:
acknowledge(stop)
return {**self.report(), 'terminal': {'runtime_stop': None if stop is None else vars(stop),
'external_failure': external_failure}}
class Observer:
def __init__(self, *args, **kwargs):
pass
def __enter__(self):
events.append('observer_enter')
return self
def __exit__(self, *_):
events.append('observer_exit')
def supplemental(self, state):
assert 'observer_enter' in events
return 'snapshot'
def infer(x, kind):
events.append('inference')
if fault.phase == 'inference':
raise ValueError('bad inference')
if fault.phase == 'signal':
raise InterruptedError('signal')
return 'actual_action', None, None
def verify(row):
assert cfg.seed == protocol.trial(row).seed
events.append(f'verify_{row}')
if cfg.mutation or fault.phase == 'identity' and row == fault.row:
raise ValueError('actual config changed')
def matrix(proto, rows):
lookup = {r['trial']['index']: r for r in rows}
return {'rows': [lookup.get(t.index, {'status': 'not_run'}) for t in proto.trials]}
api = NS(hooks=Hooks, recorder=Recorder, environment=Env, instrumentation=Observer,
backend_rng=lambda _: None, infer=infer, matrix=matrix)
budget = NS(check=lambda **_: None, recording=lambda *_: None, before_step=lambda _: None)
return NS(api=api, cfg=cfg, protocol=protocol, budget=budget, verify=verify,
events=events, fault=fault, state=state,
run=lambda indices: cli.execute_lifecycle(api, cfg, protocol, indices, tmp_path, budget, verify))
def test_pair_mismatch_stops_before_reference_inference_or_step(lifecycle):
lifecycle.fault.phase = 'pair_identity'
report = lifecycle.run(list(range(20)))
assert report['failures'][0]['context']['message'] == 'identity_pair_initial_state_mismatch'
assert 'reset_1' in lifecycle.events and 'step_1' not in lifecycle.events
assert lifecycle.events.count('inference') == 1
assert report['rows'][2]['status'] == 'not_run'
assert not report['continuation_eligible']
def test_twenty_rows_continue_only_nonfatal_durable_order(lifecycle):
f = lifecycle
result = f.run(list(range(20)))
assert result['continuation_eligible'] and not result['failures']
assert not result['complete_backend_state_equality'] and not result['deterministic_replay']
assert result['equality_scope'] == 'recorded_application_controller_and_enumerated_observable_rng_only'
assert result['observed_counts'] == {'attempted_steps': 41, 'completed_steps': 41,
'initialization_steps': 21, 'rollout_steps': 20}
assert f.events.count('construct') == 1 and f.events.count('observer_enter') == 1
assert f.events.index('hooks_enter') < f.events.index('construct') < f.events.index('observer_enter')
for row in range(19):
assert f.events.index(f'durable_{row}') < f.events.index(f'ack_{row}') < f.events.index(f'reset_{row+1}')
assert f.events[-4:] == ['observer_exit', 'env_close', 'hooks_exit', 'verify_19']
assert not any(row['functional_passed'] for row in result['rows']) # functional fail retained
@pytest.mark.parametrize('phase', ['construct', 'inference', 'control', 'after_step', 'finalize', 'signal'])
def test_external_failures_no_fake_actions_ticks_or_continuation(lifecycle, phase):
f = lifecycle
f.fault.phase = phase
result = f.run(list(range(20)))
assert not result['continuation_eligible'] and result['failures']
assert all(row['status'] == 'not_run' for row in result['rows'][1:])
if phase not in ('after_step', 'finalize'):
assert 'step_0' not in f.events
if phase == 'finalize':
assert 'ack_0' not in f.events and result['rows'][0]['status'] == 'persistence_failed'
if phase == 'signal':
assert result['failures'][0]['causes'] == ['process_signal_external_failure']
assert f.events[-1] == 'verify_0'
@pytest.mark.parametrize('fatal,ack', [(True, True), (False, False)])
def test_no_transition_after_fatal_or_unacknowledged(lifecycle, fatal, ack):
lifecycle.fault.fatal, lifecycle.fault.acknowledge = fatal, ack
result = lifecycle.run(list(range(20)))
assert 'reset_1' not in lifecycle.events
assert not result['continuation_eligible']
def test_postcleanup_same_cfg_detects_mutation_no_renormalizing(lifecycle):
lifecycle.fault.mutate = True
actual = lifecycle.cfg
result = lifecycle.run([0])
assert lifecycle.cfg is actual and actual.mutation
assert not result['continuation_eligible']
assert result['failures'][-1]['causes'] == ['identity_external_failure']
def test_transition_identity_failure_leaves_later_rows_not_run(lifecycle):
lifecycle.fault.phase, lifecycle.fault.row = 'identity', 1
result = lifecycle.run(list(range(20)))
assert 'reset_1' not in lifecycle.events and not result['continuation_eligible']
assert result['rows'][1]['status'] == 'not_run'
def test_forwarders_never_silently_invent_missing_snapshots():
forward = cli.ForwardingEvidence()
for function, args in ((forward, (None, None)), (forward.seam, (None, None)),
(forward.supplemental, (None,))):
with pytest.raises(RuntimeError, match='uninstalled'):
function(*args)
def test_check_only_is_stdlib_no_runtime_or_launch(monkeypatch, capsys):
monkeypatch.setattr(cli, 'real_runtime', lambda *_: pytest.fail('Kit must not load'))
assert cli.main(['--check-only']) == 0
result = json.loads(capsys.readouterr().out)
assert len(result['rows']) == 2
assert cli.SCRIPT in result['future_sources']['runner']
assert result['actual_backend_imports_and_performance'] == 'unqualified'
def test_prepare_only_normalized_binding_never_constructor(tmp_path, monkeypatch):
events = []
fake = NS(binding=NS(FORMAT='test-v4', PROVENANCE_SCOPE='development', PROVENANCE_LIMITATIONS=(),
prepare_binding=lambda **_: b'{"imports":{}}',
verify_binding=lambda *_, **__: None, incidental_observations=lambda _: {}),
arguments={'dependency_inventory': {'modules': {}}},
app=NS(close=lambda **_: events.append('app_close')))
monkeypatch.setattr(cli, 'real_runtime', lambda *_: fake)
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
monkeypatch.setattr(cli, 'runtime_metadata', lambda _: {})
args = NS(output=tmp_path / 'prepare', inventory=tmp_path / 'inventory', trial_index=0, artifact_root=tmp_path)
assert cli.prepare(args) == 0
assert (args.output / 'binding.json').read_bytes() == b'{"imports":{}}'
report = json.loads((args.output / 'preparation.json').read_bytes())
assert not report['execution_authorized'] and report['sim_step_calls'] == 0
assert events == ['app_close']
def test_missing_storage_qualification_blocks_before_reservation(tmp_path, monkeypatch):
monkeypatch.setattr(cli, 'real_runtime', lambda *_: pytest.fail('no runtime'))
args = NS(storage_policy=None, trusted_storage_policy_sha256=None, artifact_root=tmp_path)
with pytest.raises(RuntimeError, match='bound_policy_unavailable'):
cli.supervise(args)
assert list(tmp_path.iterdir()) == []
def trusted_inputs(tmp_path):
root = tmp_path / 'campaign'
root.mkdir()
inventory, manifest = tmp_path / 'inventory.json', tmp_path / 'manifest.json'
inventory.write_text('{}')
manifest.write_text('{}')
review = {'format': 'skillfactory.output-storage-policy.v4', 'artifact_root': str(root),
'limit_bytes': b.CAP_BYTES, 'scope': b.STORAGE_SCOPE,
'approval_sha256': cli.STORAGE_APPROVAL_SHA256, 'native_paths': cli.native_paths(root),
'shared_native_paths': cli.shared_native_paths(),
'task_owned_cache_paths': cli.task_owned_cache_paths(root),
'free_space_paths': cli.free_space_paths(root),
'free_space_mount_identities': {p: cli.mount_identity(Path(p)) for p in cli.free_space_paths(root)},
'check_interval_seconds': 1, 'max_native_growth_bytes': b.CAP_BYTES,
'min_free_bytes': 2 * b.CAP_BYTES, 'coverage_gaps': list(cli.NATIVE_GAPS),
'native_hard_cap': False, 'boot_id': b.boot_id(),
'mount_identity': cli.mount_identity(root), 'not_after_boottime': b.boottime() + 3500}
review_path = tmp_path / 'storage.json'
b.durable_write(review_path, b.encode(review))
args = NS(protocol=cli.PROTOCOLS[0], artifact_root=root, inventory=inventory, manifest=manifest,
device='cpu', trusted_manifest_sha256=b.digest(manifest), storage_policy=review_path,
trusted_storage_policy_sha256=b.digest(review_path), grant=tmp_path / 'grant.json')
grant = {'format': 'skillfactory.evaluation-parent-grant.v2', 'campaign_id': 'CPU-test-only',
'launch_index': 0, 'stage': 'P1', 'trial_indices': [0],
'manifest_sha256': args.trusted_manifest_sha256, 'protocol_sha256': b.digest(ROOT / args.protocol),
'inventory_sha256': b.digest(inventory), 'artifact_root': str(root), 'boot_id': b.boot_id(),
'not_after_boottime': b.boottime() + 1000, 'parent_review': 'CPU fixture not an actual grant',
'p1_evidence_sha256': [], 'storage_policy_sha256': args.trusted_storage_policy_sha256,
'storage_scope': b.STORAGE_SCOPE,
'launch_identity_sha256': cli.hashlib.sha256(b.encode(cli.launch_identity())).hexdigest()}
b.durable_write(args.grant, b.encode(grant))
args.trusted_grant_sha256 = b.digest(args.grant)
return args, grant, review
@pytest.mark.parametrize('field,value', [('native_hard_cap', True),
('scope', 'monitored_only'), ('boot_id', 'old'),
('not_after_boottime', 0), ('mount_identity', 'other')])
def test_storage_policy_only_trusts_bound_facts_not_capacity(tmp_path, field, value):
args, _, review = trusted_inputs(tmp_path)
review[field] = value
args.storage_policy.write_bytes(b.encode(review))
args.trusted_storage_policy_sha256 = b.digest(args.storage_policy)
with pytest.raises(RuntimeError, match='storage_policy'):
cli.storage_policy_check(args, b)
def test_no_self_trusted_manifest_digest(tmp_path):
args, grant, _ = trusted_inputs(tmp_path)
args.trusted_manifest_sha256 = '0' * 64
with pytest.raises(ValueError):
cli.grant_check(grant, args, b)
def test_failed_popen_is_charged_and_not_retryable(tmp_path, monkeypatch):
args, _, _ = trusted_inputs(tmp_path)
calls = []
monkeypatch.setattr(cli, 'native_size', lambda path, **_kw: native_scan_result(0, path=path))
def failed(*a, **kw):
calls.append(a)
raise OSError('CPU mock Popen failure')
monkeypatch.setattr(cli.subprocess, 'Popen', failed)
monkeypatch.setattr(cli.ctypes, 'CDLL', lambda *_a, **_kw: NS(prctl=lambda *_: 0))
assert cli.supervise(args) == 1
ledger = json.loads((args.artifact_root / 'budget.json').read_bytes())
assert ledger['launches'][0]['reserved_physical_calls'] == 14500
assert ledger['launches'][0]['observed_counts'] is None
assert not ledger['launches'][0]['continuation_eligible']
assert len(calls) == 1 and calls[0][0][0] == '/usr/bin/bash'
assert cli.supervise(args) == 1
assert len(calls) == 1 # same grant cannot relaunch
def test_worker_handoff_requires_live_parent_socket():
left, right = cli.socket.socketpair()
try:
with pytest.raises(RuntimeError, match='parent_supervisor'):
cli.worker_payload(right.detach())
finally:
left.close()
right.close()
def test_private_worker_requires_reservation_and_single_consumption(tmp_path):
(tmp_path / 'launch-0').mkdir()
row = reservation()
b.durable_write(tmp_path / 'budget.json', b.encode({'launches': [row]}))
b.durable_write(tmp_path / 'launch-0/worker-claim.json', b'consumed')
payload = {'launch_identity': cli.launch_identity(), 'reservation': row,
'arguments': {'artifact_root': str(tmp_path), 'inventory': 'x',
'manifest': 'y'}}
row['grant']['trial_indices'] = [0]
# Changed reservation is not authorized by immutable ledger bytes.
with pytest.raises(RuntimeError, match='stale_reservation'):
cli.run_worker(payload)
def test_single_row_unacknowledged_is_not_eligible(lifecycle):
lifecycle.fault.acknowledge = False
assert not lifecycle.run([0])['continuation_eligible']
def test_supervisor_three_fresh_processes_bound_logs_and_P2_review(tmp_path, monkeypatch):
args, grant, _ = trusted_inputs(tmp_path)
calls, payloads, cleaned = [], [], []
class Channel:
def fileno(self):
return 987
def set_inheritable(self, _):
pass
def close(self):
pass
class Owned:
supervisor = (1, 2)
known = set()
unknown = False
def attach(self, _):
pass
def discover(self):
return {}
def exchange(child, channel, payload, *_):
payloads.append(payload)
def popen(command, **kwargs):
index = len(calls)
calls.append((command, kwargs))
output = args.artifact_root / f'launch-{index}'
b.durable_write(output / 'result.json', b.encode({
'observed_counts': {'attempted_steps': 7, 'completed_steps': 7,
'initialization_steps': 2, 'rollout_steps': 5},
'continuation_eligible': True, 'preclose_identity_verified': True}))
b.durable_write(output / 'preclose-filesystem.json', b.encode({
'files': {}, 'frozen_root': str(tmp_path / 'no-frozen'), 'scope': 'CPU fixture'}))
stdout = tmp_path / f'mock-pipe-{index}'
stdout.write_bytes(b'CPU mocked child output\n')
return NS(pid=12345, stdout=stdout.open('rb'), returncode=0, wait=lambda **_: None, poll=lambda: 0)
monkeypatch.setattr(cli.socket, 'socketpair', lambda: (Channel(), Channel()))
monkeypatch.setattr(cli.subprocess, 'Popen', popen)
monkeypatch.setattr(cli, 'exchange_worker', exchange)
monkeypatch.setattr(cli, 'OwnedProcesses', Owned)
monkeypatch.setattr(cli, 'native_size', lambda path, **_kw: native_scan_result(0, path=path))
monkeypatch.setattr(cli.ctypes, 'CDLL', lambda *_a, **_kw: NS(prctl=lambda *_: 0))
monkeypatch.setattr(cli.os, 'getpgid', lambda _: 12345)
monkeypatch.setattr(cli, 'cleanup_group', lambda child, *_: cleaned.append(child.pid) or True)
for index in range(3):
args.protocol = cli.PROTOCOLS[index == 2]
grant.update(launch_index=index, stage='P2' if index == 2 else 'P1',
trial_indices=list(range(20)) if index == 2 else [index],
protocol_sha256=b.digest(ROOT / args.protocol))
if index == 2:
grant['p1_evidence_sha256'] = [b.tree_digest(args.artifact_root / f'launch-{i}') for i in (0, 1)]
args.grant.write_bytes(b.encode(grant))
args.trusted_grant_sha256 = b.digest(args.grant)
assert cli.supervise(args) == 0
assert len(calls) == len(payloads) == len(cleaned) == 3
assert [p['reservation']['grant']['trial_indices'] for p in payloads] == [[0], [1], list(range(20))]
assert all(c[0][0] == '/usr/bin/bash' and c[1]['start_new_session'] for c in calls)
assert all(c[1]['pass_fds'] == (987,) for c in calls)
ledger = json.loads((args.artifact_root / 'budget.json').read_bytes())
assert sum(row['reserved_physical_calls'] for row in ledger['launches']) == 120000
assert sum(row['observed_counts']['completed_steps'] for row in ledger['launches']) == 21
assert (args.artifact_root / 'launch-2/stdout.log').read_bytes() == b'CPU mocked child output\n'
def test_worker_single_consumption_even_with_valid_reservation(tmp_path):
row = reservation(clock=b.boottime())
row['grant'].update(boot_id=b.boot_id(), trial_indices=[0])
(tmp_path / 'launch-0').mkdir()
b.durable_write(tmp_path / 'budget.json', b.encode({'launches': [row]}))
b.durable_write(tmp_path / 'launch-0/worker-claim.json', b'already consumed')
payload = {'launch_identity': cli.launch_identity(), 'reservation': row,
'arguments': {'artifact_root': str(tmp_path), 'inventory': 'x',
'manifest': 'y', 'protocol': cli.PROTOCOLS[0]}}
with pytest.raises(FileExistsError):
cli.run_worker(payload)
@pytest.mark.parametrize('protocol_index,rows', [(0, 2), (1, 20), (2, 20)])
def test_worker_runtime_failure_reports_not_run_and_durable_failure(tmp_path, monkeypatch, protocol_index, rows):
args, grant, _ = trusted_inputs(tmp_path)
args.protocol = cli.PROTOCOLS[protocol_index]
row = reservation(clock=b.boottime())
row['grant'] = grant
(args.artifact_root / 'launch-0').mkdir()
b.durable_write(args.artifact_root / 'budget.json', b.encode({'launches': [row]}))
payload = {'launch_identity': cli.launch_identity(), 'reservation': row, 'arguments': {
'artifact_root': str(args.artifact_root), 'inventory': str(args.inventory),
'manifest': str(args.manifest), 'trusted_manifest_sha256': args.trusted_manifest_sha256,
'protocol': args.protocol}}
def fail(*_):
raise RuntimeError('mock unsupported import layout')
monkeypatch.setattr(cli, 'real_runtime', fail)
assert cli.run_worker(payload) == 1
result = json.loads((args.artifact_root / 'launch-0/result.json').read_bytes())
assert result['observed_counts'] is None and not result['continuation_eligible']
assert len(result['rows']) == rows and all(r['status'] == 'not_run' for r in result['rows'])
assert result['failures']
def test_source_qualified_provider_reads_real_RNG_without_sampling():
import random
import numpy as np
import torch
path = ROOT / cli.PACKAGE / 'tasks/manager_based/contact_cycle_sim/evaluation_rng_scope.py'
spec = importlib.util.spec_from_file_location('test_source_qualified_rng_scope', path)
scope = importlib.util.module_from_spec(spec)
spec.loader.exec_module(scope)
before = torch.get_rng_state().clone(), random.getstate(), np.random.get_state()
snapshot = scope.backend_rng(NS(env=NS(device='cpu')))
assert snapshot['coverage']['torch_task_global']['state']['cpu'] == before[0].tolist()
assert torch.equal(before[0], torch.get_rng_state()) and random.getstate() == before[1]
after = np.random.get_state()
assert np.array_equal(before[2][1], after[1]) and before[2][2:] == after[2:]
assert scope.GENERATOR_INVENTORY['additional_generators'] == []
assert 'even with zero' in scope.GENERATOR_INVENTORY['task_sampling']
assert all(value == 'unobserved' for value in snapshot['unobserved_internals'].values())
assert not any(snapshot['evidence_scope']['claims'].values())
def test_actual_runtime_hooks_recorder_partial_stop_through_runner(tmp_path, monkeypatch):
"""Real evaluation loop/recorder with CPU backend accessors, never Kit imports."""
import types
import torch
if 'SkillFactory' not in sys.modules:
package = types.ModuleType('SkillFactory')
package.__path__ = [str(ROOT / cli.PACKAGE)]
monkeypatch.setitem(sys.modules, 'SkillFactory', package)
from test_contact_cycle_sim_evaluation_runtime import runtime as runtime_fixture
fixture = runtime_fixture.__wrapped__(monkeypatch)
runtime = next(fixture)
try:
recording = __import__(cli.TASK + 'evaluation_recording', fromlist=['EvaluationRecorder'])
audit = __import__(cli.LEG + 'evaluation_audit', fromlist=['matrix_report'])
# Physical accessors and manager adapter only; actual evidence is copied by
# the real recorder, not manufactured by this test or runner.
monkeypatch.setattr(runtime.base.CycleState, 'friction', lambda _: torch.zeros(1, 3))
monkeypatch.setattr(runtime.base.CycleState, 'slip_velocity', lambda _: torch.zeros(1))
closed, constructed = [], []
def environment(**kw):
env = runtime.env.EvaluationContactCycleEnv(**kw)
env.action_manager.get_term = lambda _: env.term
env._cycle_state.safety_causes = lambda: (
('cpu_functional_stop',) if int(env._cycle_state.substeps[0]) >= 2 else ())
env.close = lambda: closed.append(True)
constructed.append(env)
return env
class Observer:
def __init__(self, env, **_):
self.env = env
def __enter__(self):
assert not self.env._started
return self
def __exit__(self, *_):
pass
def supplemental(self, s):
return {'actuator_state': {'cpu_mock': 'explicit test backend, no physical claim'},
'rng': {'cpu_mock': [1]}, 'applied': None}
p = protocols.load_protocol(ROOT / cli.PROTOCOLS[0])
cfg = runtime.config()
cfg.seed = p.trial(0).seed
seen_cfg = []
api = NS(
environment=environment, recorder=recording.EvaluationRecorder,
hooks=lambda **kw: runtime.env.RuntimeHooks(runtime.Sim, [(runtime.Robot, 'reset'),
(runtime.Robot, 'write_root_pose_to_sim'), (runtime.Scene, 'reset')], **kw),
instrumentation=Observer, backend_rng=lambda _: None, matrix=audit.matrix_report,
infer=lambda x, _: (torch.zeros(1, 3), torch.zeros(1, 3), torch.ones(3) * .1))
budget = NS(check=lambda **_: None, recording=lambda *_: None, before_step=lambda _: None)
result = cli.execute_lifecycle(api, cfg, p, [0], tmp_path, budget,
lambda _: seen_cfg.append(cfg))
assert closed == [True] and all(c is constructed[0].cfg for c in seen_cfg)
assert result['observed_counts']['initialization_steps'] == 2
assert result['observed_counts']['rollout_steps'] == 2
assert result['rows'][0]['observed_physics_ticks'] == 2
assert not result['rows'][0]['functional_passed']
assert (tmp_path / 'trial-00/metrics.json').exists()
import time
reader_spec = importlib.util.spec_from_file_location(
'evaluation_lifecycle_trace_reader', ROOT / 'scripts/audit_cycle_sim_evaluation.py')
reader = importlib.util.module_from_spec(reader_spec)
reader_spec.loader.exec_module(reader)
trace = [json.loads(line) for line in reader.trace_lines(
tmp_path / 'trial-00' / recording.TRACE_FILENAME, compressed=True,
max_bytes=8 * 1024 * 1024, max_record_bytes=1024 * 1024,
max_decoded_bytes=8 * 1024 * 1024, deadline=time.monotonic() + 30)]
assert [r['kind'] for r in trace] == ['initial', 'control', 'tick', 'tick', 'step_result', 'terminal']
assert trace[-2]['reward'] is None and trace[-2]['ticks_advanced'] == 2
finally:
with pytest.raises(StopIteration):
next(fixture)
def test_historic_evidence_paths_are_protected():
for path in (ROOT, ROOT / 'logs/contact_cycle/frozen_replay_p0_01/new',
ROOT / 'logs/contact_cycle/sim_residual_training_v1/new',
ROOT / 'releases/new'):
with pytest.raises(ValueError, match='historic'):
cli.protect_artifact_path(path)
cli.protect_artifact_path(ROOT / 'logs/contact_cycle/evaluation_runs_v1/future-campaign')
def test_provider_accepts_actual_normalized_resolvable_PD_without_calling_it(monkeypatch):
import ast
from types import ModuleType
source = Path('/home/cen/IsaacLab/source/isaaclab/isaaclab/utils/string.py').read_text()
node = next(n for n in ast.parse(source).body if isinstance(n, ast.ClassDef) and n.name == 'ResolvableString')
namespace = {}
tree = ast.Module(body=[ast.ImportFrom(module='__future__', names=[ast.alias(name='annotations')], level=0), node],
type_ignores=[])
exec(compile(ast.fix_missing_locations(tree), 'pinned-ResolvableString-CPU-extraction', 'exec'), namespace)
resolver = namespace['ResolvableString']
class PD:
def __init__(self):
pytest.fail('Qualification must never construct actuator')
class Cfg:
pass
for module, symbol, value in (
('isaaclab.actuators.actuator_pd', 'IdealPDActuator', PD),
('isaaclab.actuators.actuator_pd_cfg', 'IdealPDActuatorCfg', Cfg),
('isaaclab.utils.string', 'ResolvableString', resolver),
):
mock = ModuleType(module)
setattr(mock, symbol, value)
monkeypatch.setitem(sys.modules, module, mock)
path = ROOT / cli.PACKAGE / 'tasks/manager_based/contact_cycle_sim/evaluation_rng_scope.py'
spec = importlib.util.spec_from_file_location('provider_qualification_CPU_test', path)
scope = importlib.util.module_from_spec(spec)
spec.loader.exec_module(scope)
actuator = Cfg()
actuator.class_type = resolver('isaaclab.actuators.actuator_pd:IdealPDActuator')
cfg = NS(scene=NS(num_envs=1, robot=NS(actuators={'leg': actuator})), load_randomization=None,
observations=NS(policy=NS(enable_corruption=False)), cycle_parameters={'sensor': {'noise': 0}})
with pytest.raises(ValueError):
scope.qualify_config(cfg) # empty cache cannot be silently resolved
actuator.class_type._resolved_callable = PD
assert scope.qualify_config(cfg)['additional_generators'] == []
actuator.class_type._resolve_error = ValueError('cached error')
with pytest.raises(ValueError):
scope.qualify_config(cfg)
def test_before_Kit_close_verifies_actual_row_not_mutated_seed(tmp_path, monkeypatch):
args, grant, _ = trusted_inputs(tmp_path)
protocol = protocols.load_protocol(ROOT / cli.PROTOCOLS[1])
row = reservation(index=2, clock=b.boottime())
grant.update(boot_id=b.boot_id(), trial_indices=list(range(20)), launch_index=2)
row['grant'] = grant
(args.artifact_root / 'launch-2').mkdir()
b.durable_write(args.artifact_root / 'budget.json', b.encode({'launches': [row]}))
payload = {'launch_identity': cli.launch_identity(), 'reservation': row, 'arguments': {
'artifact_root': str(args.artifact_root), 'inventory': str(args.inventory),
'manifest': str(args.manifest), 'trusted_manifest_sha256': args.trusted_manifest_sha256,
'protocol': cli.PROTOCOLS[1]}}
cfg = NS(seed=protocol.trial(0).seed, sim=NS(device='cpu'))
verified = []
def verify(_manifest, *, trial_index, **_):
verified.append(trial_index)
if cfg.seed != protocol.trial(trial_index).seed:
raise ValueError('Unexpected actual cfg seed mutation before app close')
policy = NS()
policy.to = lambda _: policy
runtime = NS(cfg=cfg, protocol=protocol, api=NS(), arguments={'dependency_inventory': {'modules': {}}},
binding=NS(verify_binding=verify),
replay=NS(verify_bundle=lambda _: ({}, {}), CANDIDATE='unused',
load_frozen_policy=lambda *_: policy),
app=NS(close=lambda **_: None))
def execute(*_):
cfg.seed = protocol.trial(0).seed
return {'continuation_eligible': True, 'failures': [], 'observed_counts': None, 'last_trial_index': 19}
monkeypatch.setattr(cli, 'real_runtime', lambda *_: runtime)
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
monkeypatch.setattr(cli, 'runtime_metadata', lambda _: {})
monkeypatch.setattr(cli, 'execute_lifecycle', execute)
assert cli.run_worker(payload) == 1
result = json.loads((args.artifact_root / 'launch-2/result.json').read_bytes())
assert verified == [0, 19] and not result['continuation_eligible']
assert result['failures'][-1]['causes'] == ['identity_external_failure']
def test_signal_cleanup_uses_reserve_then_kills_only_owned_descendant(monkeypatch):
clock = NS(now=0., alive=True)
signals = []
def kill(pid, sig):
assert pid == 123456
signals.append(sig)
if sig == cli.signal.SIGKILL:
clock.alive = False
owned = NS(discover=lambda: {123456: (1, 2, 3, 'S')} if clock.alive else {},
signal=lambda pid, row, sig: kill(pid, sig), reap=lambda _: None, unknown=False)
monkeypatch.setattr(cli.time, 'sleep', lambda dt: setattr(clock, 'now', clock.now + dt))
assert cli.cleanup_group(NS(pid=123456, poll=lambda: None), 120, NS(boottime=lambda: clock.now), owned)
assert signals == [cli.signal.SIGTERM, cli.signal.SIGKILL]
assert 110 <= clock.now < 120
def test_native_outputs_excluded_from_owned_cap_but_not_owned_traces(tmp_path, fake_clock):
(tmp_path / 'native').mkdir()
(tmp_path / 'native/cache').write_bytes(b'x' * 4096)
(tmp_path / 'trace').write_bytes(b'x' * 123)
assert b.artifact_size(tmp_path, exclude_native=True) == 123
assert b.artifact_size(tmp_path) == 4219
budget = b.LaunchBudget(tmp_path, reservation(), clock=fake_clock.clock, boot=fake_clock.boot_fn)
budget.check(pending=b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE - 123)
with pytest.raises(RuntimeError, match='storage'):
budget.check(pending=b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE - 122)
@pytest.mark.parametrize('field,value', [('approval_sha256', '0' * 64), ('check_interval_seconds', 2),
('coverage_gaps', []), ('native_paths', []),
('shared_native_paths', []), ('task_owned_cache_paths', []),
('format', 'skillfactory.output-storage-policy.v2'),
('format', 'skillfactory.output-storage-policy.v3'),
('free_space_paths', []), ('free_space_mount_identities', {}),
('min_free_bytes', 0), ('max_native_growth_bytes', 0)])
def test_storage_policy_requires_explicit_approved_monitoring(tmp_path, field, value):
args, _, policy = trusted_inputs(tmp_path)
policy[field] = value
args.storage_policy.write_bytes(b.encode(policy))
args.trusted_storage_policy_sha256 = b.digest(args.storage_policy)
with pytest.raises(RuntimeError, match='storage_policy'):
cli.storage_policy_check(args, b)
def native_scan_result(size, free=10 * b.CAP_BYTES, *, path=None):
return {'logical_bytes': size, 'permission_gaps': [], 'skipped_count': 0,
'free_space': {'free_bytes': free, 'mount_identity': cli.mount_identity(Path(path)) if path else None},
'symlink_paths': [], 'disappeared_paths': []}
def test_native_monitor_baseline_interval_growth_and_free_space(tmp_path, monkeypatch):
args, _, policy = trusted_inputs(tmp_path)
clock = NS(now=0, size=10, free=10 * b.CAP_BYTES)
def scan(path, **_kw):
result = native_scan_result(clock.size, clock.free)
result['free_space']['mount_identity'] = cli.mount_identity(Path(path))
return result
monkeypatch.setattr(cli, 'native_size', scan)
monitor = cli.NativeMonitor(policy, NS(boottime=lambda: clock.now, STORAGE_SCOPE=b.STORAGE_SCOPE))
assert set(monitor.baseline.values()) == {10}
clock.size = b.CAP_BYTES
monitor.check() # not yet next sample
clock.now = 1
with pytest.raises(RuntimeError, match='growth_stop'):
monitor.check()
clock.size = 10
clock.free = b.CAP_BYTES
with pytest.raises(RuntimeError, match='free_space_stop'):
monitor.check(force=True)
assert monitor.report()['native_hard_cap'] is False
def test_grant_binds_launch_and_output_only_scope(tmp_path):
args, grant, _ = trusted_inputs(tmp_path)
cli.grant_check(grant, args, b)
for field in ('launch_identity_sha256', 'storage_policy_sha256', 'storage_scope'):
with pytest.raises(ValueError, match='mismatch'):
cli.grant_check({**grant, field: 'different'}, args, b)
def test_launch_bridge_fails_closed_on_source_or_environment_change(monkeypatch):
monkeypatch.setattr(b, 'digest', lambda _: '0' * 64)
with pytest.raises(RuntimeError, match='launch_source_changed'):
cli.launch_command([], {})
monkeypatch.undo()
with pytest.raises(RuntimeError, match='no_virtualenv'):
cli.launch_command([], {'CONDA_PREFIX': '/another/python'})
def test_owned_processes_rejects_preexisting_child_tree(monkeypatch):
parent = cli.os.getpid()
monkeypatch.setattr(cli, 'process_table', lambda: {parent: (1, 0, 0, 'S'), 90: (2, parent, 90, 'S')})
with pytest.raises(RuntimeError, match='preexisting_children'):
cli.OwnedProcesses()
@pytest.mark.parametrize('unrelated_adoption', [False, True])
def test_cleanup_rediscovery_staged_adoption_group_escape_and_unrelated_tree(monkeypatch, unrelated_adoption):
parent = cli.os.getpid()
table = {parent: (1, 0, 0, 'S'), 90: (2, 999, 90, 'S')}
monkeypatch.setattr(cli, 'process_table', lambda: dict(table))
owned = cli.OwnedProcesses()
table[100] = (10, parent, 100, 'S')
child = NS(pid=100, poll=lambda: None)
owned.attach(child)
clock = NS(now=0)
signaled = []
def send(pid, row, sig):
signaled.append((pid, sig))
assert pid != 90
if sig == cli.signal.SIGKILL:
del table[pid]
# Each descendant is first observed after its parent is gone; every
# one escaped the original process group before adoption.
if pid < 102:
table[pid + 1] = (row[0] + 1, parent, pid + 1, 'S')
if unrelated_adoption:
table[90] = (2, parent, 90, 'S') # excluded preexisting unrelated identity
monkeypatch.setattr(owned, 'signal', send)
monkeypatch.setattr(cli.time, 'sleep', lambda dt: setattr(clock, 'now', clock.now + dt))
assert cli.cleanup_group(child, 12, NS(boottime=lambda: clock.now), owned) is not unrelated_adoption
assert owned.unknown is unrelated_adoption # unexpected adoption is never relabeled as clean
assert [pid for pid, sig in signaled if sig == cli.signal.SIGKILL] == [100, 101, 102]
assert table[90][0] == 2 and clock.now < 12
def test_owned_pidfd_rechecks_identity_after_open(monkeypatch):
parent = cli.os.getpid()
table = {parent: (1, 0, 0, 'S')}
monkeypatch.setattr(cli, 'process_table', lambda: dict(table))
owned = cli.OwnedProcesses()
table[100] = (10, parent, 100, 'S')
owned.attach(NS(pid=100))
closed = []
def opened(_):
table[100] = (999, 555, 555, 'S') # PID reused before pidfd open
return 987
monkeypatch.setattr(cli.os, 'pidfd_open', opened)
monkeypatch.setattr(cli.os, 'close', closed.append)
monkeypatch.setattr(cli.signal, 'pidfd_send_signal', lambda *_: pytest.fail('must not signal reused PID'))
owned.signal(100, (10, parent, 100, 'S'), cli.signal.SIGKILL)
assert closed == [987] and not owned.discover()
def test_cleanup_deadline_reports_remaining_owned_processes(monkeypatch):
clock = NS(now=0)
signals = []
owned = NS(discover=lambda: {100: (10, 1, 100, 'S')},
signal=lambda pid, row, sig: signals.append(sig), reap=lambda _: None, unknown=False)
monkeypatch.setattr(cli.time, 'sleep', lambda dt: setattr(clock, 'now', clock.now + dt))
assert not cli.cleanup_group(NS(pid=100), .2, NS(boottime=lambda: clock.now), owned)
assert signals == [cli.signal.SIGKILL] and .2 <= clock.now < .3
def test_stdout_capture_startup_traceback_and_overflow(tmp_path):
source = tmp_path / 'pipe'
traceback = b'Traceback: startup failed before handshake\n'
source.write_bytes(traceback + b'x' * (b.STDOUT_BYTES + 100))
capture = cli.StdoutCapture(source.open('rb'), tmp_path / 'stdout.log', b)
capture.pump()
with pytest.raises(RuntimeError, match='capture_overflow'):
capture.check()
capture.close()
log = (tmp_path / 'stdout.log').read_bytes()
assert log.startswith(traceback) and len(log) == b.STDOUT_BYTES
assert capture.discarded == len(traceback) + 100
@pytest.mark.parametrize('exit_code', [1, None])
def test_handshake_early_exit_or_timeout_keeps_startup_output(tmp_path, monkeypatch, exit_code):
source = tmp_path / 'pipe'
source.write_bytes(b'Traceback: no handshake\n')
capture = cli.StdoutCapture(source.open('rb'), tmp_path / 'stdout.log', b)
clock = NS(now=0)
row = reservation(clock=0)
budget = NS(reservation=row, check=lambda: None)
owned = NS(discover=lambda: {})
monitor = NS(check=lambda: None)
monkeypatch.setattr(cli.time, 'sleep', lambda dt: setattr(clock, 'now', clock.now + dt))
left, right = cli.socket.socketpair()
try:
with pytest.raises(RuntimeError, match='exited_without_handshake|handshake_timeout'):
cli.exchange_worker(NS(poll=lambda: exit_code), left, {}, budget, capture, owned, monitor,
NS(boottime=lambda: clock.now, CLEANUP_SECONDS=120))
finally:
left.close()
right.close()
capture.close()
assert (tmp_path / 'stdout.log').read_bytes() == b'Traceback: no handshake\n'
def test_harmless_installed_launch_probe(tmp_path):
"""Opt-in ONE <=60s stdlib/CLI exec-chain probe, never runner main/Kit.
The parent -c harness carries the exact supervisor script/--run identity;
the child calls the real worker_payload authentication but not run_worker.
"""
if cli.os.environ.get('SKILLFACTORY_FD_PROBE') != '1':
pytest.skip('explicit harmless installed shell/FD probe opt-in required')
worker = r'''
import json, os, runpy, sys
from pathlib import Path
m = runpy.run_path(sys.argv[1], run_name='fd_probe_worker')
payload = m['worker_payload'](int(sys.argv[2]))
assert payload['probe'] == 'no-kit'
assert Path(sys.executable).resolve() == Path(m['RESOLVED_PYTHON']).resolve()
assert not any(x.split('.')[0] in ('omni', 'isaacsim', 'torch', 'pxr') for x in sys.modules)
print(json.dumps({'handshake': 'authenticated', 'pid': os.getpid(),
'executable': str(Path(sys.executable).resolve()),
'ISAACLAB_PATH': os.environ['ISAACLAB_PATH'],
'CARB_APP_PATH': os.environ['CARB_APP_PATH'],
'EXP_PATH': os.environ['EXP_PATH'],
'RESOURCE_NAME': os.environ['RESOURCE_NAME'],
'PYTHONPATH': os.environ['PYTHONPATH'],
'no_application_modules': True}), flush=True)
'''
harness = r'''
import ctypes, json, os, runpy, socket, subprocess, sys, time
from pathlib import Path
from types import SimpleNamespace as NS
m = runpy.run_path(sys.argv[1], run_name='fd_probe_supervisor')
b = m['cpu_module']('evaluation_budget')
owned = m['OwnedProcesses']()
assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0
root = Path(sys.argv[3])
left, right = socket.socketpair()
env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1')
for key in ('CONDA_PREFIX', 'VIRTUAL_ENV', 'PYTHONEXE', 'PYTHONHOME', 'args'):
env.pop(key, None)
command = m['launch_command'](['-c', sys.argv[4], sys.argv[1], str(right.fileno())], env)
child = subprocess.Popen(command, env=env, cwd=m['ROOT'], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, start_new_session=True, pass_fds=(right.fileno(),))
right.close()
capture = m['StdoutCapture'](child.stdout, root / 'probe.stdout.log', b)
owned.attach(child)
started = b.boottime()
budget = b.LaunchBudget(root, {'deadline_boottime': started + 160,
'grant': {'boot_id': b.boot_id()}})
clean = False
try:
m['exchange_worker'](child, left, {'probe': 'no-kit'}, budget, capture, owned, NS(check=lambda: None), b)
while not capture.eof or child.poll() is None:
budget.check()
capture.check()
owned.discover()
time.sleep(.01)
finally:
clean = m['cleanup_group'](child, min(started + 50, b.boottime() + 10), b, owned, capture.pump)
capture.close()
left.close()
print(json.dumps({'returncode': child.returncode, 'cleanup_clean': clean,
'owned_remaining': owned.discover(), 'seconds': b.boottime() - started,
'stdout_bytes': capture.written, 'stdout_discarded': capture.discarded}))
assert child.returncode == 0 and clean and not capture.discarded
'''
result = cli.subprocess.run(['/usr/bin/python3.12', '-c', harness, str(ROOT / cli.SCRIPT), '--run',
str(tmp_path), worker], capture_output=True, text=True, timeout=60)
log = (tmp_path / 'probe.stdout.log').read_text()
print('HARMLESS PROBE supervisor:', result.stdout.strip())
print('HARMLESS PROBE child:', log.strip())
assert result.returncode == 0, result.stderr + result.stdout + log
report = json.loads(result.stdout.strip().splitlines()[-1])
assert report['cleanup_clean'] and report['owned_remaining'] == {} and report['seconds'] < 60
child_report = json.loads(log.strip().splitlines()[-1])
assert child_report['handshake'] == 'authenticated' and child_report['no_application_modules']
assert '/home/cen/IsaacLab/source/isaaclab' in child_report['PYTHONPATH']
assert 'Traceback' not in log and report['stdout_bytes'] < 16384
@pytest.mark.parametrize('overflow', [False, True])
def test_supervise_captures_failed_handshake_through_cleanup(tmp_path, monkeypatch, overflow):
args, _, _ = trusted_inputs(tmp_path)
monkeypatch.setattr(cli, 'native_size', lambda path, **_kw: native_scan_result(0, path=path))
monkeypatch.setattr(cli.ctypes, 'CDLL', lambda *_a, **_kw: NS(prctl=lambda *_: 0))
owned = NS(supervisor=(1, 2), known={(100, 10)}, attach=lambda _: None,
discover=lambda: {}, reap=lambda _: None, unknown=False)
monkeypatch.setattr(cli, 'OwnedProcesses', lambda: owned)
source = tmp_path / 'mock-pipe'
trace = b'Traceback: import failed BEFORE hello\n'
source.write_bytes(trace + (b'x' * b.STDOUT_BYTES if overflow else b''))
def popen(*_, **__):
return NS(pid=100, stdout=source.open('rb'), returncode=1, poll=lambda: 1)
def failed_exchange(child, channel, payload, budget, capture, *_):
if overflow:
capture.check()
capture.check()
raise RuntimeError('identity_worker_handshake_eof')
monkeypatch.setattr(cli.subprocess, 'Popen', popen)
monkeypatch.setattr(cli, 'exchange_worker', failed_exchange)
assert cli.supervise(args) == 1
output = args.artifact_root / 'launch-0'
log = (output / 'stdout.log').read_bytes()
assert log.startswith(trace) and len(log) <= b.STDOUT_BYTES
result = json.loads((output / 'supervisor.json').read_bytes())
assert result['cleanup_clean'] and not result['continuation_eligible']
assert result['storage']['scope'] == b.STORAGE_SCOPE
assert bool(result['stdout_discarded_bytes']) is overflow
ledger = json.loads((args.artifact_root / 'budget.json').read_bytes())
assert ledger['launches'][0]['reserved_physical_calls'] == 14500
assert ledger['launches'][0]['status'] == 'closed'
def test_unrelated_descendant_exclusion_survives_parent_exit(monkeypatch):
parent = cli.os.getpid()
table = {parent: (1, 0, 0, 'S'), 90: (2, 999, 90, 'S')}
monkeypatch.setattr(cli, 'process_table', lambda: dict(table))
owned = cli.OwnedProcesses()
table[100] = (10, parent, 100, 'S')
owned.attach(NS(pid=100))
table[91] = (3, 90, 91, 'S') # unrelated descendant born after baseline
assert set(owned.discover()) == {100}
del table[90]
table[91] = (3, parent, 91, 'S')
assert set(owned.discover()) == {100} and owned.unknown
@pytest.mark.parametrize('failure', ['capture', 'attach'])
def test_supervise_setup_failure_terminates_reaps_and_closes_pipes(tmp_path, monkeypatch, failure):
args, _, _ = trusted_inputs(tmp_path)
monkeypatch.setattr(cli, 'native_size', lambda path, **_kw: native_scan_result(0, path=path))
monkeypatch.setattr(cli.ctypes, 'CDLL', lambda *_a, **_kw: NS(prctl=lambda *_: 0))
parent = cli.os.getpid()
table = {parent: (1, 0, 0, 'S')}
monkeypatch.setattr(cli, 'process_table', lambda: dict(table))
owned = cli.OwnedProcesses()
monkeypatch.setattr(cli, 'OwnedProcesses', lambda: owned)
events = []
pipe = (tmp_path / 'pipe').open('w+b')
child = NS(pid=100, stdout=pipe, returncode=None)
def poll():
if table.get(100, (None, None, None, None))[3] == 'Z':
del table[100]
child.returncode = -cli.signal.SIGTERM
events.append('reaped')
return child.returncode
child.poll = poll
def popen(*_, **kwargs):
assert kwargs['start_new_session']
table[100] = (10, parent, 100, 'S')
events.append('spawned')
return child
attach = owned.attach
def attaching(process):
assert events == ['spawned']
events.append('attached')
if failure == 'attach':
raise RuntimeError('injected_attach_failure')
attach(process)
def capture(*_):
assert owned.root == (100, 10)
events.append('capture')
raise OSError('injected_capture_constructor_failure')
def terminate(pid, row, sig):
assert pid == 100 and row[0] == 10 and pipe.closed
assert sig == cli.signal.SIGTERM
table[100] = (10, parent, 100, 'Z')
events.append('terminated')
monkeypatch.setattr(cli.subprocess, 'Popen', popen)
monkeypatch.setattr(owned, 'attach', attaching)
monkeypatch.setattr(owned, 'signal', terminate)
monkeypatch.setattr(cli, 'StdoutCapture', capture)
assert cli.supervise(args) == 1
assert events == ['spawned', 'attached'] + (['capture'] if failure == 'capture' else []) + [
'terminated', 'reaped']
assert pipe.closed and not owned.discover()
result = json.loads((args.artifact_root / 'launch-0/supervisor.json').read_bytes())
assert result['cleanup_clean'] is (failure == 'capture')
assert result['ownership_unknown'] is (failure == 'attach')
assert result['returncode'] == -cli.signal.SIGTERM
assert not result['continuation_eligible']
assert 'injected_' in result['errors'][0]['context']['message']
def test_stdout_capture_configuration_failure_closes_log_and_pipe(tmp_path, monkeypatch):
pipe = (tmp_path / 'pipe').open('w+b')
log = (tmp_path / 'stdout.log').open('xb')
monkeypatch.setattr(cli.Path, 'open', lambda *_a, **_kw: log)
def fail(*_):
raise OSError('injected_nonblocking_failure')
monkeypatch.setattr(cli.os, 'set_blocking', fail)
with pytest.raises(OSError, match='injected_nonblocking_failure'):
cli.StdoutCapture(pipe, tmp_path / 'stdout.log', b)
assert log.closed and pipe.closed
@pytest.mark.parametrize('arguments', [['--run'], ['--check-only'], ['--_worker-fd', '3'],
['--prepare-binding', '--inventory', '/supplied']])
def test_inventory_discovery_cli_restricted_to_preparation(monkeypatch, arguments):
monkeypatch.setattr(cli, 'prepare', lambda *_: pytest.fail('must reject before preparation'))
monkeypatch.setattr(cli, 'supervise', lambda *_: pytest.fail('must reject before execution'))
with pytest.raises(SystemExit) as error:
cli.main([*arguments, '--discover-inventory'])
assert error.value.code == 2
@pytest.mark.parametrize('discovery_failure', [False, True])
def test_prepare_discovery_proposal_exclusive_and_fail_closed(tmp_path, monkeypatch, discovery_failure):
events = []
proposal = {'scope': 'test', 'modules': {'test': {}}, 'artifacts': {'native': {}}}
fake = NS(cfg=object(), binding=NS(FORMAT='test-v4', PROVENANCE_SCOPE='development', PROVENANCE_LIMITATIONS=(),
prepare_binding=lambda **_: b'{"imports":{}}',
verify_binding=lambda *_, **__: None, incidental_observations=lambda _: {}),
arguments={'dependency_inventory': {'modules': {}}},
app=NS(close=lambda **_: events.append('app_close')))
monkeypatch.setattr(cli, 'real_runtime', lambda *_: fake)
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
monkeypatch.setattr(cli, 'runtime_metadata', lambda _: {})
def discover(*_):
if discovery_failure:
raise ValueError('Missing reviewed dependency modules')
return proposal
monkeypatch.setattr(cli, 'proposed_inventory', discover)
args = NS(output=tmp_path / 'prepare', inventory=None, trial_index=0,
artifact_root=tmp_path, discover_inventory=True)
if discovery_failure:
with pytest.raises(ValueError, match='Missing reviewed dependency'):
cli.prepare(args)
assert not (args.output / 'binding.json').exists()
else:
assert cli.prepare(args) == 0
assert json.loads((args.output / 'inventory.proposed.json').read_bytes()) == proposal
report = json.loads((args.output / 'preparation.json').read_bytes())
assert report['inventory_status'] == 'unreviewed_proposal' and not report['execution_authorized']
assert events == ['app_close']
with pytest.raises(FileExistsError):
cli.prepare(args) # never overwrite a prior attempt
assert events == ['app_close']
def test_proposed_inventory_uses_unchanged_binding_coverage_and_rejects_missing_native(tmp_path, monkeypatch):
# Reuse file-backed synthetic numerical fixtures, never actual numerical imports.
from test_contact_cycle_sim_evaluation_binding import binding, numerical_inventory
for name in tuple(sys.modules):
if name.split('.')[0] in binding.NUMERICAL_ROOTS:
monkeypatch.delitem(sys.modules, name)
numerical_inventory(tmp_path, monkeypatch)
monkeypatch.setattr(binding, '_sources', lambda *_: ({}, set()))
monkeypatch.setattr(binding, '_encode', lambda *_: None)
monkeypatch.setattr(binding, '_imports', lambda *_, **__: {})
proposal = cli.proposed_inventory(binding, object())
assert binding._dependency_inventory(proposal, set(), set()) == proposal
module = sys.modules['warp._src.context']
monkeypatch.delitem(sys.modules, 'warp._src.context')
with pytest.raises(ValueError, match='warp._src.context'):
cli.proposed_inventory(binding, object())
monkeypatch.setitem(sys.modules, 'warp._src.context', module)
(tmp_path / 'site/warp/bin/synthetic.so').unlink()
with pytest.raises(ValueError, match='missing numerical native layout'):
cli.proposed_inventory(binding, object())
def test_storage_policy_historical_v2_is_not_reinterpreted(tmp_path):
args, _, policy = trusted_inputs(tmp_path)
policy['format'] = 'skillfactory.output-storage-policy.v2'
del policy['shared_native_paths'], policy['task_owned_cache_paths']
policy['coverage_gaps'] = list(cli.NATIVE_GAPS[:-1])
args.storage_policy.write_bytes(b.encode(policy))
args.trusted_storage_policy_sha256 = b.digest(args.storage_policy)
with pytest.raises(RuntimeError, match='storage_policy_identity_or_conditions_mismatch'):
cli.storage_policy_check(args, b)
@pytest.mark.parametrize('field', ['native_paths', 'shared_native_paths', 'task_owned_cache_paths', 'free_space_paths'])
@pytest.mark.parametrize('change', ['omission', 'addition', 'alias'])
def test_storage_policy_paths_are_exact_writer_paths(tmp_path, change, field):
args, _, policy = trusted_inputs(tmp_path)
if change == 'omission':
policy[field].pop()
elif change == 'addition':
policy[field].append('/tmp/arbitrary-external-cache')
else:
policy[field][0] += '/../alias'
args.storage_policy.write_bytes(b.encode(policy))
args.trusted_storage_policy_sha256 = b.digest(args.storage_policy)
with pytest.raises(RuntimeError, match='storage_policy_identity_or_conditions_mismatch'):
cli.storage_policy_check(args, b)
def test_storage_policy_installed_data_logs_and_campaign_native_stay_strict(tmp_path):
paths = cli.native_paths(tmp_path)
shared = cli.shared_native_paths()
assert set(shared) < set(paths)
for strict in (str(tmp_path / 'native'), '/home/cen/isaacsim-6.0.1/data',
'/home/cen/isaacsim-6.0.1/kit/data', '/home/cen/isaacsim-6.0.1/kit/logs'):
assert strict in paths and strict not in shared
assert all(Path(p).is_relative_to(tmp_path / 'native') for p in cli.task_owned_cache_paths(tmp_path))
@pytest.mark.parametrize('missing', ['omni.replicator', 'omni.replicator.core'])
def test_preimport_replicator_allows_only_intended_absence(monkeypatch, missing):
def absent(name):
assert name == cli.OPTIONAL_REPLICATOR
raise ModuleNotFoundError('optional absent', name=missing)
monkeypatch.setattr(cli.importlib, 'import_module', absent)
report = cli.preimport_replicator()
assert report == {'module': 'omni.replicator.core', 'available': False,
'missing_module': missing, 'sampling_scope': 'no configured sampling',
'scope_requires_nominal_config_qualification': True,
'complete_rng_state_claim': False}
@pytest.mark.parametrize('error', [ModuleNotFoundError('required parent', name='omni'),
ModuleNotFoundError('transitive', name='unrelated_dependency'),
ModuleNotFoundError('broken internal', name='omni.replicator.broken'),
ModuleNotFoundError('unspecified'),
AttributeError('broken extension'), ImportError('broken native')])
def test_preimport_replicator_propagates_import_defects(monkeypatch, error):
def broken(_):
raise error
monkeypatch.setattr(cli.importlib, 'import_module', broken)
with pytest.raises(type(error)) as caught:
cli.preimport_replicator()
assert caught.value is error
@pytest.mark.parametrize('available', [False, True])
def test_replicator_availability_must_match_bound_inventory(monkeypatch, available):
monkeypatch.delitem(sys.modules, cli.OPTIONAL_REPLICATOR, raising=False)
if available:
monkeypatch.setitem(sys.modules, cli.OPTIONAL_REPLICATOR, NS())
matching = {'modules': {cli.OPTIONAL_REPLICATOR: {}} if available else {}}
cli.check_replicator_inventory(matching)
with pytest.raises(RuntimeError, match='optional_replicator_availability_changed'):
cli.check_replicator_inventory({'modules': {} if available else {cli.OPTIONAL_REPLICATOR: {}}})
def test_present_replicator_source_not_recursive_metadata_in_proposed_inventory(tmp_path, monkeypatch):
from test_contact_cycle_sim_evaluation_binding import binding, load, numerical_inventory
for name in tuple(sys.modules):
if name.split('.')[0] in binding.NUMERICAL_ROOTS:
monkeypatch.delitem(sys.modules, name)
numerical_inventory(tmp_path, monkeypatch)
dependency_path = tmp_path / 'replicator_dependency.py'
dependency_path.write_text('def seed(value):\n return value\n')
dependency = load('_replicator_dependency_test', dependency_path, monkeypatch)
module_path = tmp_path / 'replicator_core.py'
module_path.write_text('from _replicator_dependency_test import seed\n')
module = load(cli.OPTIONAL_REPLICATOR, module_path, monkeypatch)
monkeypatch.setattr(binding, '_sources', lambda *_: ({}, set()))
monkeypatch.setattr(binding, '_encode', lambda *_: None)
monkeypatch.setattr(binding, '_imports', lambda *_, **__: {})
report = cli.preimport_replicator()
assert report['available'] and report['missing_module'] is None
assert report['sampling_scope'] == 'no configured sampling' and not report['complete_rng_state_claim']
proposal = cli.proposed_inventory(binding, object())
assert dependency.__name__ not in proposal['modules']
for actual, path in ((module, module_path),):
assert proposal['modules'][actual.__name__] == {
'path': str(path), 'sha256': binding.sha256(path.read_bytes())}
assert binding._dependency_inventory(proposal, set(), set()) == proposal
cli.check_replicator_inventory(proposal)
def test_proposed_inventory_aggregates_independent_invalid_entries(tmp_path, monkeypatch):
from test_contact_cycle_sim_evaluation_binding import binding, load, numerical_inventory
for name in tuple(sys.modules):
if name.split('.')[0] in binding.NUMERICAL_ROOTS:
monkeypatch.delitem(sys.modules, name)
numerical_inventory(tmp_path, monkeypatch)
source = tmp_path / 'provider.py'
source.write_text('')
provider = load('diagnostic_provider', source, monkeypatch)
for name in ('broken_provider_one', 'broken_provider_two'):
module = cli.ModuleType(name)
monkeypatch.setitem(sys.modules, name, module)
setattr(provider, name, module)
monkeypatch.setattr(binding, '_sources', lambda *_: ({}, set()))
monkeypatch.setattr(binding, '_encode', lambda *_: None)
monkeypatch.setattr(binding, '_imports', lambda *_, **__: {
'diagnostic_provider': {'path': str(source), 'sha256': binding.sha256(source.read_bytes())}})
monkeypatch.setattr(binding, 'REQUIRED_RUNTIME', {**binding.REQUIRED_RUNTIME,
**{n: ('test-required', '/missing', '0' * 64)
for n in ('broken_provider_one', 'broken_provider_two')}})
(tmp_path / 'site/warp/bin/synthetic.so').unlink()
(tmp_path / 'site/torch/lib/synthetic.so').unlink()
monkeypatch.setattr(binding, '_dependency_inventory', lambda *_: pytest.fail('invalid proposal accepted'))
with pytest.raises(cli.InventoryPreparationError) as caught:
cli.proposed_inventory(binding, object())
errors = caught.value.report['invalid_entries']
assert {row['entry'] for row in errors} == {
'broken_provider_one', 'broken_provider_two', str(tmp_path / 'site/warp/bin'),
str(tmp_path / 'site/torch/lib')}
assert all('sha256' not in row for row in errors)
def test_prepare_inventory_diagnostic_is_fail_and_never_manifest(tmp_path, monkeypatch):
events = []
fake = NS(cfg=object(), binding=NS(FORMAT='test-v4', PROVENANCE_SCOPE='development', PROVENANCE_LIMITATIONS=(),
prepare_binding=lambda **_: pytest.fail('must not bind'),
module_diagnostic=lambda name: {'observed': name}),
arguments={}, app=NS(close=lambda **_: events.append('normal_app_close')))
monkeypatch.setattr(cli, 'real_runtime', lambda *_: fake)
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
diagnostics = [{'stage': 'dependency-module', 'entry': name, 'error': 'no real file'}
for name in ('one', 'two')]
def discover(*_):
raise cli.InventoryPreparationError(diagnostics)
monkeypatch.setattr(cli, 'proposed_inventory', discover)
args = NS(output=tmp_path / 'prepare', inventory=None, trial_index=0,
artifact_root=tmp_path, discover_inventory=True)
with pytest.raises(cli.InventoryPreparationError):
cli.prepare(args)
report = json.loads((args.output / 'inventory.diagnostic.json').read_bytes())
assert report['status'] == 'FAIL' and report['inventory_status'] == 'unreviewed_diagnostic'
assert not report['execution_authorized'] and report['invalid_entries'] == diagnostics
assert report['module_observations']['sys'] == {'observed': 'sys'}
assert sorted(p.name for p in args.output.iterdir()) == ['inventory.diagnostic.json', 'preparation-failure.json']
assert events == ['normal_app_close']
def test_constructor_failure_retains_actual_binding_diagnostic_without_preclose(lifecycle, tmp_path):
f = lifecycle
from test_contact_cycle_sim_evaluation_binding import binding
expected = binding.canonical_json({'effective_config': {'mutation': False}})
actual = binding.canonical_json({'effective_config': {'mutation': True}})
def constructor(**kwargs):
assert kwargs['cfg'] is f.cfg
f.cfg.mutation = True
raise RuntimeError('construction failed before environment returned')
def verify(row):
if f.cfg.mutation:
raise binding.BindingMismatchError(expected, actual)
f.api.environment = constructor
result = cli.execute_lifecycle(f.api, f.cfg, f.protocol, [0], tmp_path, f.budget, verify)
assert not result['continuation_eligible'] and f.cfg.mutation
assert result['failures'][0]['context']['phase'] == 'initialization'
diagnostic = result['failures'][-1]['context']['binding_diagnostic']
assert diagnostic['differences'][0]['path'] == '/effective_config/mutation'
assert diagnostic['differences'][0]['actual']['value'] is True
assert not (tmp_path / 'preclose-filesystem.json').exists()
def test_worker_persists_binding_diagnostic_before_fast_close_without_preclose(tmp_path, monkeypatch):
args, grant, _ = trusted_inputs(tmp_path)
row = reservation(clock=b.boottime())
row['grant'] = grant
output = args.artifact_root / 'launch-0'
output.mkdir()
b.durable_write(args.artifact_root / 'budget.json', b.encode({'launches': [row]}))
from test_contact_cycle_sim_evaluation_binding import binding
error = binding.BindingMismatchError(b'{"imports":{"x":1}}', b'{"imports":{"x":2}}')
closed = []
def verify(*_a, **_kw):
raise error
def close(*, exit_code):
saved = json.loads((output / 'result.json').read_bytes())
assert saved['failures'][-1]['context']['binding_diagnostic'] == error.binding_diagnostic
assert not saved['continuation_eligible'] and exit_code == 1
assert not (output / 'preclose-filesystem.json').exists()
closed.append(True)
runtime = NS(binding=NS(verify_binding=verify), arguments={'dependency_inventory': {}},
app=NS(close=close))
monkeypatch.setattr(cli, 'real_runtime', lambda *_: runtime)
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
monkeypatch.setattr(cli, 'check_replicator_inventory', lambda *_: None)
payload = {'launch_identity': cli.launch_identity(), 'reservation': row, 'arguments': {
'artifact_root': str(args.artifact_root), 'inventory': str(args.inventory),
'manifest': str(args.manifest), 'trusted_manifest_sha256': args.trusted_manifest_sha256,
'protocol': args.protocol}}
assert cli.run_worker(payload) == 1 and closed == [True]
@pytest.mark.parametrize('drift', ['constructor_config', 'unexpected_module'])
def test_immediate_postconstructor_binding_stops_before_reset_or_inference(lifecycle, drift, tmp_path):
from test_contact_cycle_sim_evaluation_binding import binding
f = lifecycle
f.fault.phase = drift
diagnostic = {'difference_count': 1, 'differences': [{'path': '/imports/isaaclab.unexpected'}]}
def verify(row):
f.verify(row)
if drift == 'unexpected_module' and 'construct' in f.events:
error = ValueError('unexpected actual module')
error.binding_diagnostic = diagnostic
raise error
result = cli.execute_lifecycle(f.api, f.cfg, f.protocol, [0], tmp_path, f.budget, verify)
assert not result['continuation_eligible']
assert result['failures'][0]['causes'] == ['identity_external_failure']
assert result['observed_counts']['initialization_steps'] == 1 # constructor only
assert result['observed_counts']['rollout_steps'] == 0
assert not {'reset_0', 'inference', 'control_0', 'step_0', 'observer_enter'} & set(f.events)
assert f.events[f.events.index('construct') + 1] == 'verify_0'
assert f.events[-3:] == ['env_close', 'hooks_exit', 'verify_0']
assert 'durable_0' in f.events
if drift == 'unexpected_module':
assert result['failures'][0]['context']['binding_diagnostic'] == diagnostic
assert result['failures'][-1]['context']['binding_diagnostic'] == diagnostic
# The real comparator includes new imports, not just the lifecycle test double.
report = binding.binding_difference(b'{"imports":{}}', b'{"imports":{"isaaclab.unexpected":"new"}}')
assert report['difference_count'] == 1
def test_external_failure_bounded_traceback_excludes_locals_and_source():
def fail(depth):
secret = 'private-local-value-not-for-artifacts' # noqa: F841 - intentional private local
if depth:
return fail(depth - 1)
raise TypeError('cannot pickle module')
try:
fail(45)
except TypeError as error:
failure = cli.external_failure('initialization', error)
encoded = json.dumps(failure)
trace = failure['context']['traceback']
assert failure['fatal'] and failure['causes'] == ['infrastructure_external_failure']
assert len(trace['frames']) == 32 and trace['omitted_outer_frames'] > 0
assert trace['frames'][-1]['function'] == 'fail'
assert trace['frames'][-1]['file'] == __file__
assert all(set(frame) == {'file', 'line', 'function'} for frame in trace['frames'])
assert 'private-local-value' not in encoded and 'raise TypeError' not in encoded
assert cli.external_failure('step', RuntimeError('unraised'))['context']['traceback'] == {
'frames': [], 'omitted_outer_frames': 0}
def test_constructor_failure_traceback_persisted_without_continuation(lifecycle):
lifecycle.fault.phase = 'construct'
result = lifecycle.run([0, 1])
failure = result['failures'][0]
assert failure['context']['phase'] == 'initialization' and failure['fatal']
frames = failure['context']['traceback']['frames']
assert [f['function'] for f in frames] == ['execute_lifecycle', '__init__']
persisted = json.loads((lifecycle.state.recorders[0].path / 'durable.json').read_bytes())
assert persisted['external'] == failure
assert not result['continuation_eligible'] and not result['accepted_skill']
assert 'reset_0' not in lifecycle.events and 'step_0' not in lifecycle.events