9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
1622 lines
78 KiB
Python
1622 lines
78 KiB
Python
"""CPU lifecycle doubles plus one explicitly harmless installed FD exec-chain probe. No Kit."""
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
spec = importlib.util.spec_from_file_location(
|
|
'evaluation_runner_test_cli', ROOT / 'scripts/evaluate_cycle_sim_residual.py')
|
|
cli = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = cli
|
|
spec.loader.exec_module(cli)
|
|
b = cli.cpu_module('evaluation_budget')
|
|
protocols = cli.cpu_module('evaluation_protocol')
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def no_real_launch(monkeypatch, request):
|
|
if request.node.name == 'test_harmless_installed_launch_probe':
|
|
return
|
|
def forbidden(*args, **kwargs):
|
|
raise AssertionError('No real launcher/process calls in runner CPU tests')
|
|
monkeypatch.setattr(cli.subprocess, 'Popen', forbidden)
|
|
if not hasattr(cli.os, 'pidfd_open'):
|
|
monkeypatch.setattr(cli.os, 'pidfd_open', forbidden, raising=False)
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_clock():
|
|
value = NS(now=100., boot='test-boot')
|
|
value.clock = lambda: value.now
|
|
value.boot_fn = lambda: value.boot
|
|
return value
|
|
|
|
|
|
def reservation(index=0, clock=100.):
|
|
ticks, rollout, seconds = b.SLOTS[index]
|
|
return {'index': index, 'status': 'reserved', 'reserved_physical_calls': ticks,
|
|
'rollout_cap': rollout, 'initialization_cap': 1000, 'started_boottime': clock,
|
|
'deadline_boottime': clock + seconds, 'grant': {'boot_id': 'test-boot'}}
|
|
|
|
|
|
def test_budget_conservative_debits_and_no_resume(tmp_path, fake_clock):
|
|
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
grant = {'launch_index': 0, 'not_after_boottime': 1000}
|
|
row = ledger.reserve(grant)
|
|
assert row['reserved_physical_calls'] == 14500 and row['observed_counts'] is None
|
|
with pytest.raises(RuntimeError, match='retry|reused'):
|
|
ledger.reserve(grant)
|
|
ledger.close() # simulate crash, no completion record
|
|
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
with pytest.raises(RuntimeError, match='interrupted'):
|
|
ledger.reserve({**grant, 'launch_index': 1})
|
|
assert ledger.data['launches'][0]['reserved_physical_calls'] == 14500
|
|
ledger.close()
|
|
|
|
|
|
def test_no_overlap_boot_change_or_campaign_substitution(tmp_path, fake_clock):
|
|
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
with pytest.raises(BlockingIOError):
|
|
b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
ledger.close()
|
|
fake_clock.boot = 'different-boot'
|
|
with pytest.raises(RuntimeError, match='boot_mismatch'):
|
|
b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
|
|
|
|
def test_three_slots_reviewed_P2_no_refunds_or_fourth(tmp_path, fake_clock):
|
|
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
for i in range(3):
|
|
if i < 2:
|
|
(tmp_path / f'launch-{i}').mkdir()
|
|
(tmp_path / f'launch-{i}' / 'result').write_text(str(i))
|
|
grant = {'launch_index': i, 'not_after_boottime': 4000,
|
|
'p1_evidence_sha256': [b.tree_digest(tmp_path / f'launch-{j}') for j in (0, 1)] if i == 2 else []}
|
|
ledger.reserve(grant)
|
|
ledger.finish(counts={'completed_steps': 1}, clean=True, eligible=True)
|
|
fake_clock.now += 10
|
|
assert sum(r['reserved_physical_calls'] for r in ledger.data['launches']) == 120000
|
|
assert all(r['observed_counts']['completed_steps'] == 1 for r in ledger.data['launches'])
|
|
with pytest.raises(RuntimeError):
|
|
ledger.reserve({'launch_index': 3})
|
|
ledger.close()
|
|
|
|
|
|
def test_cleanup_and_aggregate_boottime_not_prepare_time(tmp_path, fake_clock):
|
|
ledger = b.BudgetLedger(tmp_path, 'campaign', clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
fake_clock.now = 5000 # coding/preparation time is not charged
|
|
row = ledger.reserve({'launch_index': 0, 'not_after_boottime': 10000})
|
|
assert ledger.data['first_run_boottime'] == 5000
|
|
row['grant']['boot_id'] = 'test-boot'
|
|
budget = b.LaunchBudget(tmp_path, row, clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
fake_clock.now = 5480
|
|
with pytest.raises(RuntimeError, match='time'):
|
|
budget.check()
|
|
budget.check(cleanup=True)
|
|
ledger.finish(counts=None, clean=True, eligible=True)
|
|
fake_clock.now = 8480
|
|
with pytest.raises(RuntimeError, match='aggregate'):
|
|
ledger.reserve({'launch_index': 1, 'not_after_boottime': 10000})
|
|
ledger.close()
|
|
|
|
|
|
@pytest.mark.parametrize('mode,init,roll,attempt,allowed', [
|
|
('construction', 999, 0, 1000, True), ('initialization', 1000, 0, 1001, False),
|
|
('rollout', 0, 13499, 13500, True), ('rollout', 0, 13500, 13501, False),
|
|
('rollout', 999, 13499, 14501, False),
|
|
])
|
|
def test_all_initialization_reset_and_rollout_caps(tmp_path, fake_clock, mode, init, roll, attempt, allowed):
|
|
budget = b.LaunchBudget(tmp_path, reservation(), clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
h = NS(mode=mode, initialization_steps=init, rollout_steps=roll, attempted_steps=attempt)
|
|
if allowed:
|
|
budget.before_step(h)
|
|
else:
|
|
with pytest.raises(RuntimeError, match='physical_calls'):
|
|
budget.before_step(h)
|
|
|
|
|
|
def test_exact_pending_artifacts_headroom_signal_and_native_scope(tmp_path, fake_clock, monkeypatch):
|
|
monkeypatch.setattr(b, 'CAP_BYTES', 1000)
|
|
monkeypatch.setattr(b, 'HEADROOM_BYTES', 100)
|
|
monkeypatch.setattr(b, 'STDOUT_ALLOWANCE', 0)
|
|
(tmp_path / 'stdout.log').write_bytes(b'x' * 800)
|
|
budget = b.LaunchBudget(tmp_path, reservation(), clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
budget.recording('record', 100, 0)
|
|
with pytest.raises(RuntimeError, match='storage'):
|
|
budget.recording('record', 101, 0)
|
|
budget.recording('finalize', 200, 0)
|
|
with pytest.raises(RuntimeError, match='storage'):
|
|
budget.recording('finalize', 201, 0)
|
|
budget.interrupted = True
|
|
with pytest.raises(InterruptedError):
|
|
budget.check()
|
|
budget.check(cleanup=True)
|
|
(tmp_path / 'link').symlink_to(tmp_path / 'stdout.log')
|
|
with pytest.raises(RuntimeError, match='nonregular'):
|
|
budget.check(cleanup=True)
|
|
|
|
|
|
def test_artifact_scan_restarts_after_atomic_publication_not_partial_total(tmp_path, monkeypatch):
|
|
target = tmp_path / 'physics-counts.json'
|
|
pending = tmp_path / 'physics-counts.json.pending'
|
|
target.write_bytes(b'x')
|
|
pending.write_bytes(b'y' * 700)
|
|
scans = []
|
|
original = Path.lstat
|
|
def walk(root, **kwargs):
|
|
scans.append(True)
|
|
yield str(root), [], [target.name, pending.name] if len(scans) == 1 else [target.name]
|
|
def lstat(path, *args, **kwargs):
|
|
if path == pending:
|
|
pending.replace(target)
|
|
raise FileNotFoundError('atomic rename between listing and lstat')
|
|
return original(path, *args, **kwargs)
|
|
monkeypatch.setattr(b.os, 'walk', walk)
|
|
monkeypatch.setattr(Path, 'lstat', lstat)
|
|
assert b.artifact_size(tmp_path) == 700
|
|
assert len(scans) == 2
|
|
|
|
|
|
def test_artifact_scan_persistent_churn_is_bounded_and_permission_denied_fails(tmp_path, monkeypatch):
|
|
scans = []
|
|
def churn(root, **kwargs):
|
|
scans.append(True)
|
|
yield str(root), [], ['always-disappeared.pending']
|
|
monkeypatch.setattr(b.os, 'walk', churn)
|
|
with pytest.raises(RuntimeError, match='unstable_after_4'):
|
|
b.artifact_size(tmp_path)
|
|
assert len(scans) == 4
|
|
def inaccessible(root, **kwargs):
|
|
kwargs['onerror'](PermissionError('unreadable owned directory'))
|
|
yield # generator shape; must never yield a partial success
|
|
monkeypatch.setattr(b.os, 'walk', inaccessible)
|
|
with pytest.raises(PermissionError, match='unreadable'):
|
|
b.artifact_size(tmp_path)
|
|
|
|
|
|
@pytest.fixture(params=[1, 2])
|
|
def lifecycle(tmp_path, request): # noqa: C901 - scoped CPU doubles keep lifecycle events in one fixture
|
|
"""Deterministic environment/recorder boundaries with real runner control flow."""
|
|
events = []
|
|
protocol = protocols.load_protocol(ROOT / cli.PROTOCOLS[request.param])
|
|
cfg = NS(seed=protocol.trial(0).seed, mutation=False)
|
|
fault = NS(phase=None, row=0, fatal=False, acknowledge=True, mutate=False)
|
|
state = NS(env=None, hooks=None, recorders=[])
|
|
|
|
class Hooks:
|
|
def __init__(self, before_step):
|
|
self.before_step = before_step
|
|
self.attempted_steps = self.completed_steps = self.initialization_steps = self.rollout_steps = 0
|
|
state.hooks = self
|
|
|
|
def __enter__(self):
|
|
events.append('hooks_enter')
|
|
return self
|
|
|
|
def step(self, mode):
|
|
self.mode = mode
|
|
self.attempted_steps += 1
|
|
self.before_step(self)
|
|
self.completed_steps += 1
|
|
if mode == 'rollout':
|
|
self.rollout_steps += 1
|
|
else:
|
|
self.initialization_steps += 1
|
|
|
|
def __exit__(self, *_):
|
|
events.append('hooks_exit')
|
|
|
|
class Env:
|
|
def __init__(self, **kwargs):
|
|
assert events[-1] == 'recorder_0'
|
|
state.env = self
|
|
self.cfg = kwargs['cfg']
|
|
self.index = kwargs['trial_index']
|
|
self.evidence = kwargs['evidence_callback']
|
|
self.stop_result = None
|
|
self.hooks = kwargs['runtime_hooks']
|
|
self.acked = False
|
|
self.hooks.step('construction')
|
|
events.append('construct')
|
|
if fault.phase == 'constructor_config':
|
|
self.cfg.mutation = True
|
|
if fault.phase == 'construct':
|
|
raise RuntimeError('constructor fault')
|
|
|
|
def reset(self):
|
|
events.append(f'reset_{self.index}')
|
|
self.hooks.step('initialization')
|
|
self.evidence('initial', self)
|
|
return {'policy': 'raw46'}, {}
|
|
|
|
def step(self, action):
|
|
assert action == 'actual_action'
|
|
events.append(f'step_{self.index}')
|
|
self.hooks.step('rollout')
|
|
self.evidence('tick', self)
|
|
self.stop_result = NS(fatal=fault.fatal, causes=('functional_fail',), trial_index=self.index)
|
|
return NS(stop=self.stop_result, ticks_advanced=1, observation=None, reward=None)
|
|
|
|
def acknowledge_finalized(self, stop):
|
|
assert stop is self.stop_result
|
|
assert events[-1] == f'durable_{self.index}'
|
|
self.acked = True
|
|
events.append(f'ack_{self.index}')
|
|
|
|
def begin_trial(self, proto, index):
|
|
assert proto is protocol and index == self.index + 1 and self.acked
|
|
assert (tmp_path / f'trial-{self.index:02d}' / 'durable.json').exists()
|
|
self.index, self.acked, self.stop_result = index, False, None
|
|
return self.reset()
|
|
|
|
def close(self):
|
|
if 'observer_enter' in events:
|
|
assert events[-1] == 'observer_exit'
|
|
events.append('env_close')
|
|
if fault.mutate:
|
|
self.cfg.mutation = True
|
|
|
|
class Recorder:
|
|
def __init__(self, proto, row, path, *, supplemental, budget_check):
|
|
self.row, self.supplemental, self.path = row, supplemental, path
|
|
self.audit = NS(report=lambda: self.report())
|
|
path.mkdir()
|
|
self.stream = (path / 'partial').open('wb')
|
|
events.append(f'recorder_{row}')
|
|
state.recorders.append(self)
|
|
|
|
def __call__(self, event, env):
|
|
assert self.supplemental(env) == 'snapshot'
|
|
events.append(f'evidence_{event}_{self.row}')
|
|
return ()
|
|
|
|
def seam_callback(self, event, env):
|
|
return ()
|
|
|
|
def before_control(self, tick, x, action, **kwargs):
|
|
assert tick == 0 and x == 'raw46'
|
|
events.append(f'control_{self.row}')
|
|
return ('recording_bad_control',) if fault.phase == 'control' else ()
|
|
|
|
def after_step(self, actual):
|
|
assert actual is not None and actual.ticks_advanced == 1 and actual.reward is None
|
|
events.append(f'after_step_{self.row}')
|
|
return ('recording_bad_step',) if fault.phase == 'after_step' else ()
|
|
|
|
def report(self):
|
|
return {'trial': json.loads(json.dumps(cli.asdict(protocol.trial(self.row)))),
|
|
'initial_pair_state': {'mock_initial': self.row if fault.phase == 'pair_identity' else 0},
|
|
'functional_passed': False, 'all_causes': ['functional_fail']}
|
|
|
|
def finalize(self, *, stop, external_failure, acknowledge):
|
|
events.append(f'finalize_{self.row}')
|
|
if fault.phase == 'finalize':
|
|
raise OSError('fsync fault')
|
|
b.durable_write(self.path / 'durable.json', b.encode({'external': external_failure}))
|
|
self.stream.close()
|
|
events.append(f'durable_{self.row}')
|
|
if acknowledge and fault.acknowledge and stop is not None:
|
|
acknowledge(stop)
|
|
return {**self.report(), 'terminal': {'runtime_stop': None if stop is None else vars(stop),
|
|
'external_failure': external_failure}}
|
|
|
|
class Observer:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
events.append('observer_enter')
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
events.append('observer_exit')
|
|
|
|
def supplemental(self, state):
|
|
assert 'observer_enter' in events
|
|
return 'snapshot'
|
|
|
|
def infer(x, kind):
|
|
events.append('inference')
|
|
if fault.phase == 'inference':
|
|
raise ValueError('bad inference')
|
|
if fault.phase == 'signal':
|
|
raise InterruptedError('signal')
|
|
return 'actual_action', None, None
|
|
|
|
def verify(row):
|
|
assert cfg.seed == protocol.trial(row).seed
|
|
events.append(f'verify_{row}')
|
|
if cfg.mutation or fault.phase == 'identity' and row == fault.row:
|
|
raise ValueError('actual config changed')
|
|
|
|
def matrix(proto, rows):
|
|
lookup = {r['trial']['index']: r for r in rows}
|
|
return {'rows': [lookup.get(t.index, {'status': 'not_run'}) for t in proto.trials]}
|
|
|
|
api = NS(hooks=Hooks, recorder=Recorder, environment=Env, instrumentation=Observer,
|
|
backend_rng=lambda _: None, infer=infer, matrix=matrix)
|
|
budget = NS(check=lambda **_: None, recording=lambda *_: None, before_step=lambda _: None)
|
|
return NS(api=api, cfg=cfg, protocol=protocol, budget=budget, verify=verify,
|
|
events=events, fault=fault, state=state,
|
|
run=lambda indices: cli.execute_lifecycle(api, cfg, protocol, indices, tmp_path, budget, verify))
|
|
|
|
|
|
def test_pair_mismatch_stops_before_reference_inference_or_step(lifecycle):
|
|
lifecycle.fault.phase = 'pair_identity'
|
|
report = lifecycle.run(list(range(20)))
|
|
assert report['failures'][0]['context']['message'] == 'identity_pair_initial_state_mismatch'
|
|
assert 'reset_1' in lifecycle.events and 'step_1' not in lifecycle.events
|
|
assert lifecycle.events.count('inference') == 1
|
|
assert report['rows'][2]['status'] == 'not_run'
|
|
assert not report['continuation_eligible']
|
|
|
|
|
|
def test_twenty_rows_continue_only_nonfatal_durable_order(lifecycle):
|
|
f = lifecycle
|
|
result = f.run(list(range(20)))
|
|
assert result['continuation_eligible'] and not result['failures']
|
|
assert not result['complete_backend_state_equality'] and not result['deterministic_replay']
|
|
assert result['equality_scope'] == 'recorded_application_controller_and_enumerated_observable_rng_only'
|
|
assert result['observed_counts'] == {'attempted_steps': 41, 'completed_steps': 41,
|
|
'initialization_steps': 21, 'rollout_steps': 20}
|
|
assert f.events.count('construct') == 1 and f.events.count('observer_enter') == 1
|
|
assert f.events.index('hooks_enter') < f.events.index('construct') < f.events.index('observer_enter')
|
|
for row in range(19):
|
|
assert f.events.index(f'durable_{row}') < f.events.index(f'ack_{row}') < f.events.index(f'reset_{row+1}')
|
|
assert f.events[-4:] == ['observer_exit', 'env_close', 'hooks_exit', 'verify_19']
|
|
assert not any(row['functional_passed'] for row in result['rows']) # functional fail retained
|
|
|
|
|
|
@pytest.mark.parametrize('phase', ['construct', 'inference', 'control', 'after_step', 'finalize', 'signal'])
|
|
def test_external_failures_no_fake_actions_ticks_or_continuation(lifecycle, phase):
|
|
f = lifecycle
|
|
f.fault.phase = phase
|
|
result = f.run(list(range(20)))
|
|
assert not result['continuation_eligible'] and result['failures']
|
|
assert all(row['status'] == 'not_run' for row in result['rows'][1:])
|
|
if phase not in ('after_step', 'finalize'):
|
|
assert 'step_0' not in f.events
|
|
if phase == 'finalize':
|
|
assert 'ack_0' not in f.events and result['rows'][0]['status'] == 'persistence_failed'
|
|
if phase == 'signal':
|
|
assert result['failures'][0]['causes'] == ['process_signal_external_failure']
|
|
assert f.events[-1] == 'verify_0'
|
|
|
|
|
|
@pytest.mark.parametrize('fatal,ack', [(True, True), (False, False)])
|
|
def test_no_transition_after_fatal_or_unacknowledged(lifecycle, fatal, ack):
|
|
lifecycle.fault.fatal, lifecycle.fault.acknowledge = fatal, ack
|
|
result = lifecycle.run(list(range(20)))
|
|
assert 'reset_1' not in lifecycle.events
|
|
assert not result['continuation_eligible']
|
|
|
|
|
|
def test_postcleanup_same_cfg_detects_mutation_no_renormalizing(lifecycle):
|
|
lifecycle.fault.mutate = True
|
|
actual = lifecycle.cfg
|
|
result = lifecycle.run([0])
|
|
assert lifecycle.cfg is actual and actual.mutation
|
|
assert not result['continuation_eligible']
|
|
assert result['failures'][-1]['causes'] == ['identity_external_failure']
|
|
|
|
|
|
def test_transition_identity_failure_leaves_later_rows_not_run(lifecycle):
|
|
lifecycle.fault.phase, lifecycle.fault.row = 'identity', 1
|
|
result = lifecycle.run(list(range(20)))
|
|
assert 'reset_1' not in lifecycle.events and not result['continuation_eligible']
|
|
assert result['rows'][1]['status'] == 'not_run'
|
|
|
|
|
|
def test_forwarders_never_silently_invent_missing_snapshots():
|
|
forward = cli.ForwardingEvidence()
|
|
for function, args in ((forward, (None, None)), (forward.seam, (None, None)),
|
|
(forward.supplemental, (None,))):
|
|
with pytest.raises(RuntimeError, match='uninstalled'):
|
|
function(*args)
|
|
|
|
|
|
def test_check_only_is_stdlib_no_runtime_or_launch(monkeypatch, capsys):
|
|
monkeypatch.setattr(cli, 'real_runtime', lambda *_: pytest.fail('Kit must not load'))
|
|
assert cli.main(['--check-only']) == 0
|
|
result = json.loads(capsys.readouterr().out)
|
|
assert len(result['rows']) == 2
|
|
assert cli.SCRIPT in result['future_sources']['runner']
|
|
assert result['actual_backend_imports_and_performance'] == 'unqualified'
|
|
|
|
|
|
def test_prepare_only_normalized_binding_never_constructor(tmp_path, monkeypatch):
|
|
events = []
|
|
fake = NS(binding=NS(FORMAT='test-v4', PROVENANCE_SCOPE='development', PROVENANCE_LIMITATIONS=(),
|
|
prepare_binding=lambda **_: b'{"imports":{}}',
|
|
verify_binding=lambda *_, **__: None, incidental_observations=lambda _: {}),
|
|
arguments={'dependency_inventory': {'modules': {}}},
|
|
app=NS(close=lambda **_: events.append('app_close')))
|
|
monkeypatch.setattr(cli, 'real_runtime', lambda *_: fake)
|
|
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
|
|
monkeypatch.setattr(cli, 'runtime_metadata', lambda _: {})
|
|
args = NS(output=tmp_path / 'prepare', inventory=tmp_path / 'inventory', trial_index=0, artifact_root=tmp_path)
|
|
assert cli.prepare(args) == 0
|
|
assert (args.output / 'binding.json').read_bytes() == b'{"imports":{}}'
|
|
report = json.loads((args.output / 'preparation.json').read_bytes())
|
|
assert not report['execution_authorized'] and report['sim_step_calls'] == 0
|
|
assert events == ['app_close']
|
|
|
|
|
|
def test_missing_storage_qualification_blocks_before_reservation(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(cli, 'real_runtime', lambda *_: pytest.fail('no runtime'))
|
|
args = NS(storage_policy=None, trusted_storage_policy_sha256=None, artifact_root=tmp_path)
|
|
with pytest.raises(RuntimeError, match='bound_policy_unavailable'):
|
|
cli.supervise(args)
|
|
assert list(tmp_path.iterdir()) == []
|
|
|
|
|
|
def trusted_inputs(tmp_path):
|
|
root = tmp_path / 'campaign'
|
|
root.mkdir()
|
|
inventory, manifest = tmp_path / 'inventory.json', tmp_path / 'manifest.json'
|
|
inventory.write_text('{}')
|
|
manifest.write_text('{}')
|
|
review = {'format': 'skillfactory.output-storage-policy.v4', 'artifact_root': str(root),
|
|
'limit_bytes': b.CAP_BYTES, 'scope': b.STORAGE_SCOPE,
|
|
'approval_sha256': cli.STORAGE_APPROVAL_SHA256, 'native_paths': cli.native_paths(root),
|
|
'shared_native_paths': cli.shared_native_paths(),
|
|
'task_owned_cache_paths': cli.task_owned_cache_paths(root),
|
|
'free_space_paths': cli.free_space_paths(root),
|
|
'free_space_mount_identities': {p: cli.mount_identity(Path(p)) for p in cli.free_space_paths(root)},
|
|
'check_interval_seconds': 1, 'max_native_growth_bytes': b.CAP_BYTES,
|
|
'min_free_bytes': 2 * b.CAP_BYTES, 'coverage_gaps': list(cli.NATIVE_GAPS),
|
|
'native_hard_cap': False, 'boot_id': b.boot_id(),
|
|
'mount_identity': cli.mount_identity(root), 'not_after_boottime': b.boottime() + 3500}
|
|
review_path = tmp_path / 'storage.json'
|
|
b.durable_write(review_path, b.encode(review))
|
|
args = NS(protocol=cli.PROTOCOLS[0], artifact_root=root, inventory=inventory, manifest=manifest,
|
|
device='cpu', trusted_manifest_sha256=b.digest(manifest), storage_policy=review_path,
|
|
trusted_storage_policy_sha256=b.digest(review_path), grant=tmp_path / 'grant.json')
|
|
grant = {'format': 'skillfactory.evaluation-parent-grant.v2', 'campaign_id': 'CPU-test-only',
|
|
'launch_index': 0, 'stage': 'P1', 'trial_indices': [0],
|
|
'manifest_sha256': args.trusted_manifest_sha256, 'protocol_sha256': b.digest(ROOT / args.protocol),
|
|
'inventory_sha256': b.digest(inventory), 'artifact_root': str(root), 'boot_id': b.boot_id(),
|
|
'not_after_boottime': b.boottime() + 1000, 'parent_review': 'CPU fixture not an actual grant',
|
|
'p1_evidence_sha256': [], 'storage_policy_sha256': args.trusted_storage_policy_sha256,
|
|
'storage_scope': b.STORAGE_SCOPE,
|
|
'launch_identity_sha256': cli.hashlib.sha256(b.encode(cli.launch_identity())).hexdigest()}
|
|
b.durable_write(args.grant, b.encode(grant))
|
|
args.trusted_grant_sha256 = b.digest(args.grant)
|
|
return args, grant, review
|
|
|
|
|
|
@pytest.mark.parametrize('field,value', [('native_hard_cap', True),
|
|
('scope', 'monitored_only'), ('boot_id', 'old'),
|
|
('not_after_boottime', 0), ('mount_identity', 'other')])
|
|
def test_storage_policy_only_trusts_bound_facts_not_capacity(tmp_path, field, value):
|
|
args, _, review = trusted_inputs(tmp_path)
|
|
review[field] = value
|
|
args.storage_policy.write_bytes(b.encode(review))
|
|
args.trusted_storage_policy_sha256 = b.digest(args.storage_policy)
|
|
with pytest.raises(RuntimeError, match='storage_policy'):
|
|
cli.storage_policy_check(args, b)
|
|
|
|
|
|
def test_no_self_trusted_manifest_digest(tmp_path):
|
|
args, grant, _ = trusted_inputs(tmp_path)
|
|
args.trusted_manifest_sha256 = '0' * 64
|
|
with pytest.raises(ValueError):
|
|
cli.grant_check(grant, args, b)
|
|
|
|
|
|
def test_failed_popen_is_charged_and_not_retryable(tmp_path, monkeypatch):
|
|
args, _, _ = trusted_inputs(tmp_path)
|
|
calls = []
|
|
monkeypatch.setattr(cli, 'native_size', lambda path, **_kw: native_scan_result(0, path=path))
|
|
|
|
def failed(*a, **kw):
|
|
calls.append(a)
|
|
raise OSError('CPU mock Popen failure')
|
|
|
|
monkeypatch.setattr(cli.subprocess, 'Popen', failed)
|
|
monkeypatch.setattr(cli.ctypes, 'CDLL', lambda *_a, **_kw: NS(prctl=lambda *_: 0))
|
|
assert cli.supervise(args) == 1
|
|
ledger = json.loads((args.artifact_root / 'budget.json').read_bytes())
|
|
assert ledger['launches'][0]['reserved_physical_calls'] == 14500
|
|
assert ledger['launches'][0]['observed_counts'] is None
|
|
assert not ledger['launches'][0]['continuation_eligible']
|
|
assert len(calls) == 1 and calls[0][0][0] == '/usr/bin/bash'
|
|
assert cli.supervise(args) == 1
|
|
assert len(calls) == 1 # same grant cannot relaunch
|
|
|
|
|
|
def test_worker_handoff_requires_live_parent_socket():
|
|
left, right = cli.socket.socketpair()
|
|
try:
|
|
with pytest.raises(RuntimeError, match='parent_supervisor'):
|
|
cli.worker_payload(right.detach())
|
|
finally:
|
|
left.close()
|
|
right.close()
|
|
|
|
|
|
def test_private_worker_requires_reservation_and_single_consumption(tmp_path):
|
|
(tmp_path / 'launch-0').mkdir()
|
|
row = reservation()
|
|
b.durable_write(tmp_path / 'budget.json', b.encode({'launches': [row]}))
|
|
b.durable_write(tmp_path / 'launch-0/worker-claim.json', b'consumed')
|
|
payload = {'launch_identity': cli.launch_identity(), 'reservation': row,
|
|
'arguments': {'artifact_root': str(tmp_path), 'inventory': 'x',
|
|
'manifest': 'y'}}
|
|
row['grant']['trial_indices'] = [0]
|
|
# Changed reservation is not authorized by immutable ledger bytes.
|
|
with pytest.raises(RuntimeError, match='stale_reservation'):
|
|
cli.run_worker(payload)
|
|
|
|
|
|
def test_single_row_unacknowledged_is_not_eligible(lifecycle):
|
|
lifecycle.fault.acknowledge = False
|
|
assert not lifecycle.run([0])['continuation_eligible']
|
|
|
|
|
|
def test_supervisor_three_fresh_processes_bound_logs_and_P2_review(tmp_path, monkeypatch):
|
|
args, grant, _ = trusted_inputs(tmp_path)
|
|
calls, payloads, cleaned = [], [], []
|
|
|
|
class Channel:
|
|
def fileno(self):
|
|
return 987
|
|
|
|
def set_inheritable(self, _):
|
|
pass
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
class Owned:
|
|
supervisor = (1, 2)
|
|
known = set()
|
|
unknown = False
|
|
def attach(self, _):
|
|
pass
|
|
def discover(self):
|
|
return {}
|
|
|
|
def exchange(child, channel, payload, *_):
|
|
payloads.append(payload)
|
|
|
|
def popen(command, **kwargs):
|
|
index = len(calls)
|
|
calls.append((command, kwargs))
|
|
output = args.artifact_root / f'launch-{index}'
|
|
b.durable_write(output / 'result.json', b.encode({
|
|
'observed_counts': {'attempted_steps': 7, 'completed_steps': 7,
|
|
'initialization_steps': 2, 'rollout_steps': 5},
|
|
'continuation_eligible': True, 'preclose_identity_verified': True}))
|
|
b.durable_write(output / 'preclose-filesystem.json', b.encode({
|
|
'files': {}, 'frozen_root': str(tmp_path / 'no-frozen'), 'scope': 'CPU fixture'}))
|
|
stdout = tmp_path / f'mock-pipe-{index}'
|
|
stdout.write_bytes(b'CPU mocked child output\n')
|
|
return NS(pid=12345, stdout=stdout.open('rb'), returncode=0, wait=lambda **_: None, poll=lambda: 0)
|
|
|
|
monkeypatch.setattr(cli.socket, 'socketpair', lambda: (Channel(), Channel()))
|
|
monkeypatch.setattr(cli.subprocess, 'Popen', popen)
|
|
monkeypatch.setattr(cli, 'exchange_worker', exchange)
|
|
monkeypatch.setattr(cli, 'OwnedProcesses', Owned)
|
|
monkeypatch.setattr(cli, 'native_size', lambda path, **_kw: native_scan_result(0, path=path))
|
|
monkeypatch.setattr(cli.ctypes, 'CDLL', lambda *_a, **_kw: NS(prctl=lambda *_: 0))
|
|
monkeypatch.setattr(cli.os, 'getpgid', lambda _: 12345)
|
|
monkeypatch.setattr(cli, 'cleanup_group', lambda child, *_: cleaned.append(child.pid) or True)
|
|
for index in range(3):
|
|
args.protocol = cli.PROTOCOLS[index == 2]
|
|
grant.update(launch_index=index, stage='P2' if index == 2 else 'P1',
|
|
trial_indices=list(range(20)) if index == 2 else [index],
|
|
protocol_sha256=b.digest(ROOT / args.protocol))
|
|
if index == 2:
|
|
grant['p1_evidence_sha256'] = [b.tree_digest(args.artifact_root / f'launch-{i}') for i in (0, 1)]
|
|
args.grant.write_bytes(b.encode(grant))
|
|
args.trusted_grant_sha256 = b.digest(args.grant)
|
|
assert cli.supervise(args) == 0
|
|
assert len(calls) == len(payloads) == len(cleaned) == 3
|
|
assert [p['reservation']['grant']['trial_indices'] for p in payloads] == [[0], [1], list(range(20))]
|
|
assert all(c[0][0] == '/usr/bin/bash' and c[1]['start_new_session'] for c in calls)
|
|
assert all(c[1]['pass_fds'] == (987,) for c in calls)
|
|
ledger = json.loads((args.artifact_root / 'budget.json').read_bytes())
|
|
assert sum(row['reserved_physical_calls'] for row in ledger['launches']) == 120000
|
|
assert sum(row['observed_counts']['completed_steps'] for row in ledger['launches']) == 21
|
|
assert (args.artifact_root / 'launch-2/stdout.log').read_bytes() == b'CPU mocked child output\n'
|
|
|
|
|
|
def test_worker_single_consumption_even_with_valid_reservation(tmp_path):
|
|
row = reservation(clock=b.boottime())
|
|
row['grant'].update(boot_id=b.boot_id(), trial_indices=[0])
|
|
(tmp_path / 'launch-0').mkdir()
|
|
b.durable_write(tmp_path / 'budget.json', b.encode({'launches': [row]}))
|
|
b.durable_write(tmp_path / 'launch-0/worker-claim.json', b'already consumed')
|
|
payload = {'launch_identity': cli.launch_identity(), 'reservation': row,
|
|
'arguments': {'artifact_root': str(tmp_path), 'inventory': 'x',
|
|
'manifest': 'y', 'protocol': cli.PROTOCOLS[0]}}
|
|
with pytest.raises(FileExistsError):
|
|
cli.run_worker(payload)
|
|
|
|
|
|
@pytest.mark.parametrize('protocol_index,rows', [(0, 2), (1, 20), (2, 20)])
|
|
def test_worker_runtime_failure_reports_not_run_and_durable_failure(tmp_path, monkeypatch, protocol_index, rows):
|
|
args, grant, _ = trusted_inputs(tmp_path)
|
|
args.protocol = cli.PROTOCOLS[protocol_index]
|
|
row = reservation(clock=b.boottime())
|
|
row['grant'] = grant
|
|
(args.artifact_root / 'launch-0').mkdir()
|
|
b.durable_write(args.artifact_root / 'budget.json', b.encode({'launches': [row]}))
|
|
payload = {'launch_identity': cli.launch_identity(), 'reservation': row, 'arguments': {
|
|
'artifact_root': str(args.artifact_root), 'inventory': str(args.inventory),
|
|
'manifest': str(args.manifest), 'trusted_manifest_sha256': args.trusted_manifest_sha256,
|
|
'protocol': args.protocol}}
|
|
|
|
def fail(*_):
|
|
raise RuntimeError('mock unsupported import layout')
|
|
|
|
monkeypatch.setattr(cli, 'real_runtime', fail)
|
|
assert cli.run_worker(payload) == 1
|
|
result = json.loads((args.artifact_root / 'launch-0/result.json').read_bytes())
|
|
assert result['observed_counts'] is None and not result['continuation_eligible']
|
|
assert len(result['rows']) == rows and all(r['status'] == 'not_run' for r in result['rows'])
|
|
assert result['failures']
|
|
|
|
|
|
def test_source_qualified_provider_reads_real_RNG_without_sampling():
|
|
import random
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
path = ROOT / cli.PACKAGE / 'tasks/manager_based/contact_cycle_sim/evaluation_rng_scope.py'
|
|
spec = importlib.util.spec_from_file_location('test_source_qualified_rng_scope', path)
|
|
scope = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(scope)
|
|
before = torch.get_rng_state().clone(), random.getstate(), np.random.get_state()
|
|
snapshot = scope.backend_rng(NS(env=NS(device='cpu')))
|
|
assert snapshot['coverage']['torch_task_global']['state']['cpu'] == before[0].tolist()
|
|
assert torch.equal(before[0], torch.get_rng_state()) and random.getstate() == before[1]
|
|
after = np.random.get_state()
|
|
assert np.array_equal(before[2][1], after[1]) and before[2][2:] == after[2:]
|
|
assert scope.GENERATOR_INVENTORY['additional_generators'] == []
|
|
assert 'even with zero' in scope.GENERATOR_INVENTORY['task_sampling']
|
|
assert all(value == 'unobserved' for value in snapshot['unobserved_internals'].values())
|
|
assert not any(snapshot['evidence_scope']['claims'].values())
|
|
|
|
|
|
def test_actual_runtime_hooks_recorder_partial_stop_through_runner(tmp_path, monkeypatch):
|
|
"""Real evaluation loop/recorder with CPU backend accessors, never Kit imports."""
|
|
import types
|
|
|
|
import torch
|
|
|
|
if 'SkillFactory' not in sys.modules:
|
|
package = types.ModuleType('SkillFactory')
|
|
package.__path__ = [str(ROOT / cli.PACKAGE)]
|
|
monkeypatch.setitem(sys.modules, 'SkillFactory', package)
|
|
from test_contact_cycle_sim_evaluation_runtime import runtime as runtime_fixture
|
|
|
|
fixture = runtime_fixture.__wrapped__(monkeypatch)
|
|
runtime = next(fixture)
|
|
try:
|
|
recording = __import__(cli.TASK + 'evaluation_recording', fromlist=['EvaluationRecorder'])
|
|
audit = __import__(cli.LEG + 'evaluation_audit', fromlist=['matrix_report'])
|
|
# Physical accessors and manager adapter only; actual evidence is copied by
|
|
# the real recorder, not manufactured by this test or runner.
|
|
monkeypatch.setattr(runtime.base.CycleState, 'friction', lambda _: torch.zeros(1, 3))
|
|
monkeypatch.setattr(runtime.base.CycleState, 'slip_velocity', lambda _: torch.zeros(1))
|
|
closed, constructed = [], []
|
|
|
|
def environment(**kw):
|
|
env = runtime.env.EvaluationContactCycleEnv(**kw)
|
|
env.action_manager.get_term = lambda _: env.term
|
|
env._cycle_state.safety_causes = lambda: (
|
|
('cpu_functional_stop',) if int(env._cycle_state.substeps[0]) >= 2 else ())
|
|
env.close = lambda: closed.append(True)
|
|
constructed.append(env)
|
|
return env
|
|
|
|
class Observer:
|
|
def __init__(self, env, **_):
|
|
self.env = env
|
|
|
|
def __enter__(self):
|
|
assert not self.env._started
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
pass
|
|
|
|
def supplemental(self, s):
|
|
return {'actuator_state': {'cpu_mock': 'explicit test backend, no physical claim'},
|
|
'rng': {'cpu_mock': [1]}, 'applied': None}
|
|
|
|
p = protocols.load_protocol(ROOT / cli.PROTOCOLS[0])
|
|
cfg = runtime.config()
|
|
cfg.seed = p.trial(0).seed
|
|
seen_cfg = []
|
|
api = NS(
|
|
environment=environment, recorder=recording.EvaluationRecorder,
|
|
hooks=lambda **kw: runtime.env.RuntimeHooks(runtime.Sim, [(runtime.Robot, 'reset'),
|
|
(runtime.Robot, 'write_root_pose_to_sim'), (runtime.Scene, 'reset')], **kw),
|
|
instrumentation=Observer, backend_rng=lambda _: None, matrix=audit.matrix_report,
|
|
infer=lambda x, _: (torch.zeros(1, 3), torch.zeros(1, 3), torch.ones(3) * .1))
|
|
budget = NS(check=lambda **_: None, recording=lambda *_: None, before_step=lambda _: None)
|
|
result = cli.execute_lifecycle(api, cfg, p, [0], tmp_path, budget,
|
|
lambda _: seen_cfg.append(cfg))
|
|
assert closed == [True] and all(c is constructed[0].cfg for c in seen_cfg)
|
|
assert result['observed_counts']['initialization_steps'] == 2
|
|
assert result['observed_counts']['rollout_steps'] == 2
|
|
assert result['rows'][0]['observed_physics_ticks'] == 2
|
|
assert not result['rows'][0]['functional_passed']
|
|
assert (tmp_path / 'trial-00/metrics.json').exists()
|
|
import time
|
|
reader_spec = importlib.util.spec_from_file_location(
|
|
'evaluation_lifecycle_trace_reader', ROOT / 'scripts/audit_cycle_sim_evaluation.py')
|
|
reader = importlib.util.module_from_spec(reader_spec)
|
|
reader_spec.loader.exec_module(reader)
|
|
trace = [json.loads(line) for line in reader.trace_lines(
|
|
tmp_path / 'trial-00' / recording.TRACE_FILENAME, compressed=True,
|
|
max_bytes=8 * 1024 * 1024, max_record_bytes=1024 * 1024,
|
|
max_decoded_bytes=8 * 1024 * 1024, deadline=time.monotonic() + 30)]
|
|
assert [r['kind'] for r in trace] == ['initial', 'control', 'tick', 'tick', 'step_result', 'terminal']
|
|
assert trace[-2]['reward'] is None and trace[-2]['ticks_advanced'] == 2
|
|
finally:
|
|
with pytest.raises(StopIteration):
|
|
next(fixture)
|
|
|
|
|
|
def test_historic_evidence_paths_are_protected():
|
|
for path in (ROOT, ROOT / 'logs/contact_cycle/frozen_replay_p0_01/new',
|
|
ROOT / 'logs/contact_cycle/sim_residual_training_v1/new',
|
|
ROOT / 'releases/new'):
|
|
with pytest.raises(ValueError, match='historic'):
|
|
cli.protect_artifact_path(path)
|
|
cli.protect_artifact_path(ROOT / 'logs/contact_cycle/evaluation_runs_v1/future-campaign')
|
|
|
|
|
|
def test_provider_accepts_actual_normalized_resolvable_PD_without_calling_it(monkeypatch):
|
|
import ast
|
|
from types import ModuleType
|
|
|
|
source = Path('/home/cen/IsaacLab/source/isaaclab/isaaclab/utils/string.py').read_text()
|
|
node = next(n for n in ast.parse(source).body if isinstance(n, ast.ClassDef) and n.name == 'ResolvableString')
|
|
namespace = {}
|
|
tree = ast.Module(body=[ast.ImportFrom(module='__future__', names=[ast.alias(name='annotations')], level=0), node],
|
|
type_ignores=[])
|
|
exec(compile(ast.fix_missing_locations(tree), 'pinned-ResolvableString-CPU-extraction', 'exec'), namespace)
|
|
resolver = namespace['ResolvableString']
|
|
|
|
class PD:
|
|
def __init__(self):
|
|
pytest.fail('Qualification must never construct actuator')
|
|
|
|
class Cfg:
|
|
pass
|
|
|
|
for module, symbol, value in (
|
|
('isaaclab.actuators.actuator_pd', 'IdealPDActuator', PD),
|
|
('isaaclab.actuators.actuator_pd_cfg', 'IdealPDActuatorCfg', Cfg),
|
|
('isaaclab.utils.string', 'ResolvableString', resolver),
|
|
):
|
|
mock = ModuleType(module)
|
|
setattr(mock, symbol, value)
|
|
monkeypatch.setitem(sys.modules, module, mock)
|
|
path = ROOT / cli.PACKAGE / 'tasks/manager_based/contact_cycle_sim/evaluation_rng_scope.py'
|
|
spec = importlib.util.spec_from_file_location('provider_qualification_CPU_test', path)
|
|
scope = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(scope)
|
|
actuator = Cfg()
|
|
actuator.class_type = resolver('isaaclab.actuators.actuator_pd:IdealPDActuator')
|
|
cfg = NS(scene=NS(num_envs=1, robot=NS(actuators={'leg': actuator})), load_randomization=None,
|
|
observations=NS(policy=NS(enable_corruption=False)), cycle_parameters={'sensor': {'noise': 0}})
|
|
with pytest.raises(ValueError):
|
|
scope.qualify_config(cfg) # empty cache cannot be silently resolved
|
|
actuator.class_type._resolved_callable = PD
|
|
assert scope.qualify_config(cfg)['additional_generators'] == []
|
|
actuator.class_type._resolve_error = ValueError('cached error')
|
|
with pytest.raises(ValueError):
|
|
scope.qualify_config(cfg)
|
|
|
|
|
|
def test_before_Kit_close_verifies_actual_row_not_mutated_seed(tmp_path, monkeypatch):
|
|
args, grant, _ = trusted_inputs(tmp_path)
|
|
protocol = protocols.load_protocol(ROOT / cli.PROTOCOLS[1])
|
|
row = reservation(index=2, clock=b.boottime())
|
|
grant.update(boot_id=b.boot_id(), trial_indices=list(range(20)), launch_index=2)
|
|
row['grant'] = grant
|
|
(args.artifact_root / 'launch-2').mkdir()
|
|
b.durable_write(args.artifact_root / 'budget.json', b.encode({'launches': [row]}))
|
|
payload = {'launch_identity': cli.launch_identity(), 'reservation': row, 'arguments': {
|
|
'artifact_root': str(args.artifact_root), 'inventory': str(args.inventory),
|
|
'manifest': str(args.manifest), 'trusted_manifest_sha256': args.trusted_manifest_sha256,
|
|
'protocol': cli.PROTOCOLS[1]}}
|
|
cfg = NS(seed=protocol.trial(0).seed, sim=NS(device='cpu'))
|
|
verified = []
|
|
|
|
def verify(_manifest, *, trial_index, **_):
|
|
verified.append(trial_index)
|
|
if cfg.seed != protocol.trial(trial_index).seed:
|
|
raise ValueError('Unexpected actual cfg seed mutation before app close')
|
|
|
|
policy = NS()
|
|
policy.to = lambda _: policy
|
|
runtime = NS(cfg=cfg, protocol=protocol, api=NS(), arguments={'dependency_inventory': {'modules': {}}},
|
|
binding=NS(verify_binding=verify),
|
|
replay=NS(verify_bundle=lambda _: ({}, {}), CANDIDATE='unused',
|
|
load_frozen_policy=lambda *_: policy),
|
|
app=NS(close=lambda **_: None))
|
|
|
|
def execute(*_):
|
|
cfg.seed = protocol.trial(0).seed
|
|
return {'continuation_eligible': True, 'failures': [], 'observed_counts': None, 'last_trial_index': 19}
|
|
|
|
monkeypatch.setattr(cli, 'real_runtime', lambda *_: runtime)
|
|
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
|
|
monkeypatch.setattr(cli, 'runtime_metadata', lambda _: {})
|
|
monkeypatch.setattr(cli, 'execute_lifecycle', execute)
|
|
assert cli.run_worker(payload) == 1
|
|
result = json.loads((args.artifact_root / 'launch-2/result.json').read_bytes())
|
|
assert verified == [0, 19] and not result['continuation_eligible']
|
|
assert result['failures'][-1]['causes'] == ['identity_external_failure']
|
|
|
|
|
|
def test_signal_cleanup_uses_reserve_then_kills_only_owned_descendant(monkeypatch):
|
|
clock = NS(now=0., alive=True)
|
|
signals = []
|
|
|
|
def kill(pid, sig):
|
|
assert pid == 123456
|
|
signals.append(sig)
|
|
if sig == cli.signal.SIGKILL:
|
|
clock.alive = False
|
|
|
|
owned = NS(discover=lambda: {123456: (1, 2, 3, 'S')} if clock.alive else {},
|
|
signal=lambda pid, row, sig: kill(pid, sig), reap=lambda _: None, unknown=False)
|
|
monkeypatch.setattr(cli.time, 'sleep', lambda dt: setattr(clock, 'now', clock.now + dt))
|
|
assert cli.cleanup_group(NS(pid=123456, poll=lambda: None), 120, NS(boottime=lambda: clock.now), owned)
|
|
assert signals == [cli.signal.SIGTERM, cli.signal.SIGKILL]
|
|
assert 110 <= clock.now < 120
|
|
|
|
|
|
def test_native_outputs_excluded_from_owned_cap_but_not_owned_traces(tmp_path, fake_clock):
|
|
(tmp_path / 'native').mkdir()
|
|
(tmp_path / 'native/cache').write_bytes(b'x' * 4096)
|
|
(tmp_path / 'trace').write_bytes(b'x' * 123)
|
|
assert b.artifact_size(tmp_path, exclude_native=True) == 123
|
|
assert b.artifact_size(tmp_path) == 4219
|
|
budget = b.LaunchBudget(tmp_path, reservation(), clock=fake_clock.clock, boot=fake_clock.boot_fn)
|
|
budget.check(pending=b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE - 123)
|
|
with pytest.raises(RuntimeError, match='storage'):
|
|
budget.check(pending=b.CAP_BYTES - b.HEADROOM_BYTES - b.STDOUT_ALLOWANCE - 122)
|
|
|
|
|
|
@pytest.mark.parametrize('field,value', [('approval_sha256', '0' * 64), ('check_interval_seconds', 2),
|
|
('coverage_gaps', []), ('native_paths', []),
|
|
('shared_native_paths', []), ('task_owned_cache_paths', []),
|
|
('format', 'skillfactory.output-storage-policy.v2'),
|
|
('format', 'skillfactory.output-storage-policy.v3'),
|
|
('free_space_paths', []), ('free_space_mount_identities', {}),
|
|
('min_free_bytes', 0), ('max_native_growth_bytes', 0)])
|
|
def test_storage_policy_requires_explicit_approved_monitoring(tmp_path, field, value):
|
|
args, _, policy = trusted_inputs(tmp_path)
|
|
policy[field] = value
|
|
args.storage_policy.write_bytes(b.encode(policy))
|
|
args.trusted_storage_policy_sha256 = b.digest(args.storage_policy)
|
|
with pytest.raises(RuntimeError, match='storage_policy'):
|
|
cli.storage_policy_check(args, b)
|
|
|
|
|
|
def native_scan_result(size, free=10 * b.CAP_BYTES, *, path=None):
|
|
return {'logical_bytes': size, 'permission_gaps': [], 'skipped_count': 0,
|
|
'free_space': {'free_bytes': free, 'mount_identity': cli.mount_identity(Path(path)) if path else None},
|
|
'symlink_paths': [], 'disappeared_paths': []}
|
|
|
|
|
|
def test_native_monitor_baseline_interval_growth_and_free_space(tmp_path, monkeypatch):
|
|
args, _, policy = trusted_inputs(tmp_path)
|
|
clock = NS(now=0, size=10, free=10 * b.CAP_BYTES)
|
|
def scan(path, **_kw):
|
|
result = native_scan_result(clock.size, clock.free)
|
|
result['free_space']['mount_identity'] = cli.mount_identity(Path(path))
|
|
return result
|
|
monkeypatch.setattr(cli, 'native_size', scan)
|
|
monitor = cli.NativeMonitor(policy, NS(boottime=lambda: clock.now, STORAGE_SCOPE=b.STORAGE_SCOPE))
|
|
assert set(monitor.baseline.values()) == {10}
|
|
clock.size = b.CAP_BYTES
|
|
monitor.check() # not yet next sample
|
|
clock.now = 1
|
|
with pytest.raises(RuntimeError, match='growth_stop'):
|
|
monitor.check()
|
|
clock.size = 10
|
|
clock.free = b.CAP_BYTES
|
|
with pytest.raises(RuntimeError, match='free_space_stop'):
|
|
monitor.check(force=True)
|
|
assert monitor.report()['native_hard_cap'] is False
|
|
|
|
|
|
def test_grant_binds_launch_and_output_only_scope(tmp_path):
|
|
args, grant, _ = trusted_inputs(tmp_path)
|
|
cli.grant_check(grant, args, b)
|
|
for field in ('launch_identity_sha256', 'storage_policy_sha256', 'storage_scope'):
|
|
with pytest.raises(ValueError, match='mismatch'):
|
|
cli.grant_check({**grant, field: 'different'}, args, b)
|
|
|
|
|
|
def test_launch_bridge_fails_closed_on_source_or_environment_change(monkeypatch):
|
|
monkeypatch.setattr(b, 'digest', lambda _: '0' * 64)
|
|
with pytest.raises(RuntimeError, match='launch_source_changed'):
|
|
cli.launch_command([], {})
|
|
monkeypatch.undo()
|
|
with pytest.raises(RuntimeError, match='no_virtualenv'):
|
|
cli.launch_command([], {'CONDA_PREFIX': '/another/python'})
|
|
|
|
|
|
def test_owned_processes_rejects_preexisting_child_tree(monkeypatch):
|
|
parent = cli.os.getpid()
|
|
monkeypatch.setattr(cli, 'process_table', lambda: {parent: (1, 0, 0, 'S'), 90: (2, parent, 90, 'S')})
|
|
with pytest.raises(RuntimeError, match='preexisting_children'):
|
|
cli.OwnedProcesses()
|
|
|
|
|
|
@pytest.mark.parametrize('unrelated_adoption', [False, True])
|
|
def test_cleanup_rediscovery_staged_adoption_group_escape_and_unrelated_tree(monkeypatch, unrelated_adoption):
|
|
parent = cli.os.getpid()
|
|
table = {parent: (1, 0, 0, 'S'), 90: (2, 999, 90, 'S')}
|
|
monkeypatch.setattr(cli, 'process_table', lambda: dict(table))
|
|
owned = cli.OwnedProcesses()
|
|
table[100] = (10, parent, 100, 'S')
|
|
child = NS(pid=100, poll=lambda: None)
|
|
owned.attach(child)
|
|
clock = NS(now=0)
|
|
signaled = []
|
|
|
|
def send(pid, row, sig):
|
|
signaled.append((pid, sig))
|
|
assert pid != 90
|
|
if sig == cli.signal.SIGKILL:
|
|
del table[pid]
|
|
# Each descendant is first observed after its parent is gone; every
|
|
# one escaped the original process group before adoption.
|
|
if pid < 102:
|
|
table[pid + 1] = (row[0] + 1, parent, pid + 1, 'S')
|
|
if unrelated_adoption:
|
|
table[90] = (2, parent, 90, 'S') # excluded preexisting unrelated identity
|
|
|
|
monkeypatch.setattr(owned, 'signal', send)
|
|
monkeypatch.setattr(cli.time, 'sleep', lambda dt: setattr(clock, 'now', clock.now + dt))
|
|
assert cli.cleanup_group(child, 12, NS(boottime=lambda: clock.now), owned) is not unrelated_adoption
|
|
assert owned.unknown is unrelated_adoption # unexpected adoption is never relabeled as clean
|
|
assert [pid for pid, sig in signaled if sig == cli.signal.SIGKILL] == [100, 101, 102]
|
|
assert table[90][0] == 2 and clock.now < 12
|
|
|
|
|
|
def test_owned_pidfd_rechecks_identity_after_open(monkeypatch):
|
|
parent = cli.os.getpid()
|
|
table = {parent: (1, 0, 0, 'S')}
|
|
monkeypatch.setattr(cli, 'process_table', lambda: dict(table))
|
|
owned = cli.OwnedProcesses()
|
|
table[100] = (10, parent, 100, 'S')
|
|
owned.attach(NS(pid=100))
|
|
closed = []
|
|
def opened(_):
|
|
table[100] = (999, 555, 555, 'S') # PID reused before pidfd open
|
|
return 987
|
|
monkeypatch.setattr(cli.os, 'pidfd_open', opened)
|
|
monkeypatch.setattr(cli.os, 'close', closed.append)
|
|
monkeypatch.setattr(cli.signal, 'pidfd_send_signal', lambda *_: pytest.fail('must not signal reused PID'))
|
|
owned.signal(100, (10, parent, 100, 'S'), cli.signal.SIGKILL)
|
|
assert closed == [987] and not owned.discover()
|
|
|
|
|
|
def test_cleanup_deadline_reports_remaining_owned_processes(monkeypatch):
|
|
clock = NS(now=0)
|
|
signals = []
|
|
owned = NS(discover=lambda: {100: (10, 1, 100, 'S')},
|
|
signal=lambda pid, row, sig: signals.append(sig), reap=lambda _: None, unknown=False)
|
|
monkeypatch.setattr(cli.time, 'sleep', lambda dt: setattr(clock, 'now', clock.now + dt))
|
|
assert not cli.cleanup_group(NS(pid=100), .2, NS(boottime=lambda: clock.now), owned)
|
|
assert signals == [cli.signal.SIGKILL] and .2 <= clock.now < .3
|
|
|
|
|
|
def test_stdout_capture_startup_traceback_and_overflow(tmp_path):
|
|
source = tmp_path / 'pipe'
|
|
traceback = b'Traceback: startup failed before handshake\n'
|
|
source.write_bytes(traceback + b'x' * (b.STDOUT_BYTES + 100))
|
|
capture = cli.StdoutCapture(source.open('rb'), tmp_path / 'stdout.log', b)
|
|
capture.pump()
|
|
with pytest.raises(RuntimeError, match='capture_overflow'):
|
|
capture.check()
|
|
capture.close()
|
|
log = (tmp_path / 'stdout.log').read_bytes()
|
|
assert log.startswith(traceback) and len(log) == b.STDOUT_BYTES
|
|
assert capture.discarded == len(traceback) + 100
|
|
|
|
|
|
@pytest.mark.parametrize('exit_code', [1, None])
|
|
def test_handshake_early_exit_or_timeout_keeps_startup_output(tmp_path, monkeypatch, exit_code):
|
|
source = tmp_path / 'pipe'
|
|
source.write_bytes(b'Traceback: no handshake\n')
|
|
capture = cli.StdoutCapture(source.open('rb'), tmp_path / 'stdout.log', b)
|
|
clock = NS(now=0)
|
|
row = reservation(clock=0)
|
|
budget = NS(reservation=row, check=lambda: None)
|
|
owned = NS(discover=lambda: {})
|
|
monitor = NS(check=lambda: None)
|
|
monkeypatch.setattr(cli.time, 'sleep', lambda dt: setattr(clock, 'now', clock.now + dt))
|
|
left, right = cli.socket.socketpair()
|
|
try:
|
|
with pytest.raises(RuntimeError, match='exited_without_handshake|handshake_timeout'):
|
|
cli.exchange_worker(NS(poll=lambda: exit_code), left, {}, budget, capture, owned, monitor,
|
|
NS(boottime=lambda: clock.now, CLEANUP_SECONDS=120))
|
|
finally:
|
|
left.close()
|
|
right.close()
|
|
capture.close()
|
|
assert (tmp_path / 'stdout.log').read_bytes() == b'Traceback: no handshake\n'
|
|
|
|
|
|
def test_harmless_installed_launch_probe(tmp_path):
|
|
"""Opt-in ONE <=60s stdlib/CLI exec-chain probe, never runner main/Kit.
|
|
|
|
The parent -c harness carries the exact supervisor script/--run identity;
|
|
the child calls the real worker_payload authentication but not run_worker.
|
|
"""
|
|
if cli.os.environ.get('SKILLFACTORY_FD_PROBE') != '1':
|
|
pytest.skip('explicit harmless installed shell/FD probe opt-in required')
|
|
worker = r'''
|
|
import json, os, runpy, sys
|
|
from pathlib import Path
|
|
m = runpy.run_path(sys.argv[1], run_name='fd_probe_worker')
|
|
payload = m['worker_payload'](int(sys.argv[2]))
|
|
assert payload['probe'] == 'no-kit'
|
|
assert Path(sys.executable).resolve() == Path(m['RESOLVED_PYTHON']).resolve()
|
|
assert not any(x.split('.')[0] in ('omni', 'isaacsim', 'torch', 'pxr') for x in sys.modules)
|
|
print(json.dumps({'handshake': 'authenticated', 'pid': os.getpid(),
|
|
'executable': str(Path(sys.executable).resolve()),
|
|
'ISAACLAB_PATH': os.environ['ISAACLAB_PATH'],
|
|
'CARB_APP_PATH': os.environ['CARB_APP_PATH'],
|
|
'EXP_PATH': os.environ['EXP_PATH'],
|
|
'RESOURCE_NAME': os.environ['RESOURCE_NAME'],
|
|
'PYTHONPATH': os.environ['PYTHONPATH'],
|
|
'no_application_modules': True}), flush=True)
|
|
'''
|
|
harness = r'''
|
|
import ctypes, json, os, runpy, socket, subprocess, sys, time
|
|
from pathlib import Path
|
|
from types import SimpleNamespace as NS
|
|
m = runpy.run_path(sys.argv[1], run_name='fd_probe_supervisor')
|
|
b = m['cpu_module']('evaluation_budget')
|
|
owned = m['OwnedProcesses']()
|
|
assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0
|
|
root = Path(sys.argv[3])
|
|
left, right = socket.socketpair()
|
|
env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1')
|
|
for key in ('CONDA_PREFIX', 'VIRTUAL_ENV', 'PYTHONEXE', 'PYTHONHOME', 'args'):
|
|
env.pop(key, None)
|
|
command = m['launch_command'](['-c', sys.argv[4], sys.argv[1], str(right.fileno())], env)
|
|
child = subprocess.Popen(command, env=env, cwd=m['ROOT'], stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, start_new_session=True, pass_fds=(right.fileno(),))
|
|
right.close()
|
|
capture = m['StdoutCapture'](child.stdout, root / 'probe.stdout.log', b)
|
|
owned.attach(child)
|
|
started = b.boottime()
|
|
budget = b.LaunchBudget(root, {'deadline_boottime': started + 160,
|
|
'grant': {'boot_id': b.boot_id()}})
|
|
clean = False
|
|
try:
|
|
m['exchange_worker'](child, left, {'probe': 'no-kit'}, budget, capture, owned, NS(check=lambda: None), b)
|
|
while not capture.eof or child.poll() is None:
|
|
budget.check()
|
|
capture.check()
|
|
owned.discover()
|
|
time.sleep(.01)
|
|
finally:
|
|
clean = m['cleanup_group'](child, min(started + 50, b.boottime() + 10), b, owned, capture.pump)
|
|
capture.close()
|
|
left.close()
|
|
print(json.dumps({'returncode': child.returncode, 'cleanup_clean': clean,
|
|
'owned_remaining': owned.discover(), 'seconds': b.boottime() - started,
|
|
'stdout_bytes': capture.written, 'stdout_discarded': capture.discarded}))
|
|
assert child.returncode == 0 and clean and not capture.discarded
|
|
'''
|
|
result = cli.subprocess.run(['/usr/bin/python3.12', '-c', harness, str(ROOT / cli.SCRIPT), '--run',
|
|
str(tmp_path), worker], capture_output=True, text=True, timeout=60)
|
|
log = (tmp_path / 'probe.stdout.log').read_text()
|
|
print('HARMLESS PROBE supervisor:', result.stdout.strip())
|
|
print('HARMLESS PROBE child:', log.strip())
|
|
assert result.returncode == 0, result.stderr + result.stdout + log
|
|
report = json.loads(result.stdout.strip().splitlines()[-1])
|
|
assert report['cleanup_clean'] and report['owned_remaining'] == {} and report['seconds'] < 60
|
|
child_report = json.loads(log.strip().splitlines()[-1])
|
|
assert child_report['handshake'] == 'authenticated' and child_report['no_application_modules']
|
|
assert '/home/cen/IsaacLab/source/isaaclab' in child_report['PYTHONPATH']
|
|
assert 'Traceback' not in log and report['stdout_bytes'] < 16384
|
|
|
|
|
|
@pytest.mark.parametrize('overflow', [False, True])
|
|
def test_supervise_captures_failed_handshake_through_cleanup(tmp_path, monkeypatch, overflow):
|
|
args, _, _ = trusted_inputs(tmp_path)
|
|
monkeypatch.setattr(cli, 'native_size', lambda path, **_kw: native_scan_result(0, path=path))
|
|
monkeypatch.setattr(cli.ctypes, 'CDLL', lambda *_a, **_kw: NS(prctl=lambda *_: 0))
|
|
owned = NS(supervisor=(1, 2), known={(100, 10)}, attach=lambda _: None,
|
|
discover=lambda: {}, reap=lambda _: None, unknown=False)
|
|
monkeypatch.setattr(cli, 'OwnedProcesses', lambda: owned)
|
|
source = tmp_path / 'mock-pipe'
|
|
trace = b'Traceback: import failed BEFORE hello\n'
|
|
source.write_bytes(trace + (b'x' * b.STDOUT_BYTES if overflow else b''))
|
|
def popen(*_, **__):
|
|
return NS(pid=100, stdout=source.open('rb'), returncode=1, poll=lambda: 1)
|
|
def failed_exchange(child, channel, payload, budget, capture, *_):
|
|
if overflow:
|
|
capture.check()
|
|
capture.check()
|
|
raise RuntimeError('identity_worker_handshake_eof')
|
|
monkeypatch.setattr(cli.subprocess, 'Popen', popen)
|
|
monkeypatch.setattr(cli, 'exchange_worker', failed_exchange)
|
|
assert cli.supervise(args) == 1
|
|
output = args.artifact_root / 'launch-0'
|
|
log = (output / 'stdout.log').read_bytes()
|
|
assert log.startswith(trace) and len(log) <= b.STDOUT_BYTES
|
|
result = json.loads((output / 'supervisor.json').read_bytes())
|
|
assert result['cleanup_clean'] and not result['continuation_eligible']
|
|
assert result['storage']['scope'] == b.STORAGE_SCOPE
|
|
assert bool(result['stdout_discarded_bytes']) is overflow
|
|
ledger = json.loads((args.artifact_root / 'budget.json').read_bytes())
|
|
assert ledger['launches'][0]['reserved_physical_calls'] == 14500
|
|
assert ledger['launches'][0]['status'] == 'closed'
|
|
|
|
|
|
def test_unrelated_descendant_exclusion_survives_parent_exit(monkeypatch):
|
|
parent = cli.os.getpid()
|
|
table = {parent: (1, 0, 0, 'S'), 90: (2, 999, 90, 'S')}
|
|
monkeypatch.setattr(cli, 'process_table', lambda: dict(table))
|
|
owned = cli.OwnedProcesses()
|
|
table[100] = (10, parent, 100, 'S')
|
|
owned.attach(NS(pid=100))
|
|
table[91] = (3, 90, 91, 'S') # unrelated descendant born after baseline
|
|
assert set(owned.discover()) == {100}
|
|
del table[90]
|
|
table[91] = (3, parent, 91, 'S')
|
|
assert set(owned.discover()) == {100} and owned.unknown
|
|
|
|
|
|
@pytest.mark.parametrize('failure', ['capture', 'attach'])
|
|
def test_supervise_setup_failure_terminates_reaps_and_closes_pipes(tmp_path, monkeypatch, failure):
|
|
args, _, _ = trusted_inputs(tmp_path)
|
|
monkeypatch.setattr(cli, 'native_size', lambda path, **_kw: native_scan_result(0, path=path))
|
|
monkeypatch.setattr(cli.ctypes, 'CDLL', lambda *_a, **_kw: NS(prctl=lambda *_: 0))
|
|
parent = cli.os.getpid()
|
|
table = {parent: (1, 0, 0, 'S')}
|
|
monkeypatch.setattr(cli, 'process_table', lambda: dict(table))
|
|
owned = cli.OwnedProcesses()
|
|
monkeypatch.setattr(cli, 'OwnedProcesses', lambda: owned)
|
|
events = []
|
|
pipe = (tmp_path / 'pipe').open('w+b')
|
|
child = NS(pid=100, stdout=pipe, returncode=None)
|
|
|
|
def poll():
|
|
if table.get(100, (None, None, None, None))[3] == 'Z':
|
|
del table[100]
|
|
child.returncode = -cli.signal.SIGTERM
|
|
events.append('reaped')
|
|
return child.returncode
|
|
|
|
child.poll = poll
|
|
|
|
def popen(*_, **kwargs):
|
|
assert kwargs['start_new_session']
|
|
table[100] = (10, parent, 100, 'S')
|
|
events.append('spawned')
|
|
return child
|
|
|
|
attach = owned.attach
|
|
|
|
def attaching(process):
|
|
assert events == ['spawned']
|
|
events.append('attached')
|
|
if failure == 'attach':
|
|
raise RuntimeError('injected_attach_failure')
|
|
attach(process)
|
|
|
|
def capture(*_):
|
|
assert owned.root == (100, 10)
|
|
events.append('capture')
|
|
raise OSError('injected_capture_constructor_failure')
|
|
|
|
def terminate(pid, row, sig):
|
|
assert pid == 100 and row[0] == 10 and pipe.closed
|
|
assert sig == cli.signal.SIGTERM
|
|
table[100] = (10, parent, 100, 'Z')
|
|
events.append('terminated')
|
|
|
|
monkeypatch.setattr(cli.subprocess, 'Popen', popen)
|
|
monkeypatch.setattr(owned, 'attach', attaching)
|
|
monkeypatch.setattr(owned, 'signal', terminate)
|
|
monkeypatch.setattr(cli, 'StdoutCapture', capture)
|
|
assert cli.supervise(args) == 1
|
|
assert events == ['spawned', 'attached'] + (['capture'] if failure == 'capture' else []) + [
|
|
'terminated', 'reaped']
|
|
assert pipe.closed and not owned.discover()
|
|
result = json.loads((args.artifact_root / 'launch-0/supervisor.json').read_bytes())
|
|
assert result['cleanup_clean'] is (failure == 'capture')
|
|
assert result['ownership_unknown'] is (failure == 'attach')
|
|
assert result['returncode'] == -cli.signal.SIGTERM
|
|
assert not result['continuation_eligible']
|
|
assert 'injected_' in result['errors'][0]['context']['message']
|
|
|
|
|
|
def test_stdout_capture_configuration_failure_closes_log_and_pipe(tmp_path, monkeypatch):
|
|
pipe = (tmp_path / 'pipe').open('w+b')
|
|
log = (tmp_path / 'stdout.log').open('xb')
|
|
monkeypatch.setattr(cli.Path, 'open', lambda *_a, **_kw: log)
|
|
|
|
def fail(*_):
|
|
raise OSError('injected_nonblocking_failure')
|
|
|
|
monkeypatch.setattr(cli.os, 'set_blocking', fail)
|
|
with pytest.raises(OSError, match='injected_nonblocking_failure'):
|
|
cli.StdoutCapture(pipe, tmp_path / 'stdout.log', b)
|
|
assert log.closed and pipe.closed
|
|
|
|
|
|
@pytest.mark.parametrize('arguments', [['--run'], ['--check-only'], ['--_worker-fd', '3'],
|
|
['--prepare-binding', '--inventory', '/supplied']])
|
|
def test_inventory_discovery_cli_restricted_to_preparation(monkeypatch, arguments):
|
|
monkeypatch.setattr(cli, 'prepare', lambda *_: pytest.fail('must reject before preparation'))
|
|
monkeypatch.setattr(cli, 'supervise', lambda *_: pytest.fail('must reject before execution'))
|
|
with pytest.raises(SystemExit) as error:
|
|
cli.main([*arguments, '--discover-inventory'])
|
|
assert error.value.code == 2
|
|
|
|
|
|
@pytest.mark.parametrize('discovery_failure', [False, True])
|
|
def test_prepare_discovery_proposal_exclusive_and_fail_closed(tmp_path, monkeypatch, discovery_failure):
|
|
events = []
|
|
proposal = {'scope': 'test', 'modules': {'test': {}}, 'artifacts': {'native': {}}}
|
|
fake = NS(cfg=object(), binding=NS(FORMAT='test-v4', PROVENANCE_SCOPE='development', PROVENANCE_LIMITATIONS=(),
|
|
prepare_binding=lambda **_: b'{"imports":{}}',
|
|
verify_binding=lambda *_, **__: None, incidental_observations=lambda _: {}),
|
|
arguments={'dependency_inventory': {'modules': {}}},
|
|
app=NS(close=lambda **_: events.append('app_close')))
|
|
monkeypatch.setattr(cli, 'real_runtime', lambda *_: fake)
|
|
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
|
|
monkeypatch.setattr(cli, 'runtime_metadata', lambda _: {})
|
|
|
|
def discover(*_):
|
|
if discovery_failure:
|
|
raise ValueError('Missing reviewed dependency modules')
|
|
return proposal
|
|
|
|
monkeypatch.setattr(cli, 'proposed_inventory', discover)
|
|
args = NS(output=tmp_path / 'prepare', inventory=None, trial_index=0,
|
|
artifact_root=tmp_path, discover_inventory=True)
|
|
if discovery_failure:
|
|
with pytest.raises(ValueError, match='Missing reviewed dependency'):
|
|
cli.prepare(args)
|
|
assert not (args.output / 'binding.json').exists()
|
|
else:
|
|
assert cli.prepare(args) == 0
|
|
assert json.loads((args.output / 'inventory.proposed.json').read_bytes()) == proposal
|
|
report = json.loads((args.output / 'preparation.json').read_bytes())
|
|
assert report['inventory_status'] == 'unreviewed_proposal' and not report['execution_authorized']
|
|
assert events == ['app_close']
|
|
with pytest.raises(FileExistsError):
|
|
cli.prepare(args) # never overwrite a prior attempt
|
|
assert events == ['app_close']
|
|
|
|
|
|
def test_proposed_inventory_uses_unchanged_binding_coverage_and_rejects_missing_native(tmp_path, monkeypatch):
|
|
# Reuse file-backed synthetic numerical fixtures, never actual numerical imports.
|
|
from test_contact_cycle_sim_evaluation_binding import binding, numerical_inventory
|
|
|
|
for name in tuple(sys.modules):
|
|
if name.split('.')[0] in binding.NUMERICAL_ROOTS:
|
|
monkeypatch.delitem(sys.modules, name)
|
|
numerical_inventory(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(binding, '_sources', lambda *_: ({}, set()))
|
|
monkeypatch.setattr(binding, '_encode', lambda *_: None)
|
|
monkeypatch.setattr(binding, '_imports', lambda *_, **__: {})
|
|
proposal = cli.proposed_inventory(binding, object())
|
|
assert binding._dependency_inventory(proposal, set(), set()) == proposal
|
|
module = sys.modules['warp._src.context']
|
|
monkeypatch.delitem(sys.modules, 'warp._src.context')
|
|
with pytest.raises(ValueError, match='warp._src.context'):
|
|
cli.proposed_inventory(binding, object())
|
|
monkeypatch.setitem(sys.modules, 'warp._src.context', module)
|
|
(tmp_path / 'site/warp/bin/synthetic.so').unlink()
|
|
with pytest.raises(ValueError, match='missing numerical native layout'):
|
|
cli.proposed_inventory(binding, object())
|
|
|
|
|
|
def test_storage_policy_historical_v2_is_not_reinterpreted(tmp_path):
|
|
args, _, policy = trusted_inputs(tmp_path)
|
|
policy['format'] = 'skillfactory.output-storage-policy.v2'
|
|
del policy['shared_native_paths'], policy['task_owned_cache_paths']
|
|
policy['coverage_gaps'] = list(cli.NATIVE_GAPS[:-1])
|
|
args.storage_policy.write_bytes(b.encode(policy))
|
|
args.trusted_storage_policy_sha256 = b.digest(args.storage_policy)
|
|
with pytest.raises(RuntimeError, match='storage_policy_identity_or_conditions_mismatch'):
|
|
cli.storage_policy_check(args, b)
|
|
|
|
|
|
@pytest.mark.parametrize('field', ['native_paths', 'shared_native_paths', 'task_owned_cache_paths', 'free_space_paths'])
|
|
@pytest.mark.parametrize('change', ['omission', 'addition', 'alias'])
|
|
def test_storage_policy_paths_are_exact_writer_paths(tmp_path, change, field):
|
|
args, _, policy = trusted_inputs(tmp_path)
|
|
if change == 'omission':
|
|
policy[field].pop()
|
|
elif change == 'addition':
|
|
policy[field].append('/tmp/arbitrary-external-cache')
|
|
else:
|
|
policy[field][0] += '/../alias'
|
|
args.storage_policy.write_bytes(b.encode(policy))
|
|
args.trusted_storage_policy_sha256 = b.digest(args.storage_policy)
|
|
with pytest.raises(RuntimeError, match='storage_policy_identity_or_conditions_mismatch'):
|
|
cli.storage_policy_check(args, b)
|
|
|
|
|
|
def test_storage_policy_installed_data_logs_and_campaign_native_stay_strict(tmp_path):
|
|
paths = cli.native_paths(tmp_path)
|
|
shared = cli.shared_native_paths()
|
|
assert set(shared) < set(paths)
|
|
for strict in (str(tmp_path / 'native'), '/home/cen/isaacsim-6.0.1/data',
|
|
'/home/cen/isaacsim-6.0.1/kit/data', '/home/cen/isaacsim-6.0.1/kit/logs'):
|
|
assert strict in paths and strict not in shared
|
|
assert all(Path(p).is_relative_to(tmp_path / 'native') for p in cli.task_owned_cache_paths(tmp_path))
|
|
|
|
|
|
@pytest.mark.parametrize('missing', ['omni.replicator', 'omni.replicator.core'])
|
|
def test_preimport_replicator_allows_only_intended_absence(monkeypatch, missing):
|
|
def absent(name):
|
|
assert name == cli.OPTIONAL_REPLICATOR
|
|
raise ModuleNotFoundError('optional absent', name=missing)
|
|
|
|
monkeypatch.setattr(cli.importlib, 'import_module', absent)
|
|
report = cli.preimport_replicator()
|
|
assert report == {'module': 'omni.replicator.core', 'available': False,
|
|
'missing_module': missing, 'sampling_scope': 'no configured sampling',
|
|
'scope_requires_nominal_config_qualification': True,
|
|
'complete_rng_state_claim': False}
|
|
|
|
|
|
@pytest.mark.parametrize('error', [ModuleNotFoundError('required parent', name='omni'),
|
|
ModuleNotFoundError('transitive', name='unrelated_dependency'),
|
|
ModuleNotFoundError('broken internal', name='omni.replicator.broken'),
|
|
ModuleNotFoundError('unspecified'),
|
|
AttributeError('broken extension'), ImportError('broken native')])
|
|
def test_preimport_replicator_propagates_import_defects(monkeypatch, error):
|
|
def broken(_):
|
|
raise error
|
|
|
|
monkeypatch.setattr(cli.importlib, 'import_module', broken)
|
|
with pytest.raises(type(error)) as caught:
|
|
cli.preimport_replicator()
|
|
assert caught.value is error
|
|
|
|
|
|
@pytest.mark.parametrize('available', [False, True])
|
|
def test_replicator_availability_must_match_bound_inventory(monkeypatch, available):
|
|
monkeypatch.delitem(sys.modules, cli.OPTIONAL_REPLICATOR, raising=False)
|
|
if available:
|
|
monkeypatch.setitem(sys.modules, cli.OPTIONAL_REPLICATOR, NS())
|
|
matching = {'modules': {cli.OPTIONAL_REPLICATOR: {}} if available else {}}
|
|
cli.check_replicator_inventory(matching)
|
|
with pytest.raises(RuntimeError, match='optional_replicator_availability_changed'):
|
|
cli.check_replicator_inventory({'modules': {} if available else {cli.OPTIONAL_REPLICATOR: {}}})
|
|
|
|
|
|
def test_present_replicator_source_not_recursive_metadata_in_proposed_inventory(tmp_path, monkeypatch):
|
|
from test_contact_cycle_sim_evaluation_binding import binding, load, numerical_inventory
|
|
|
|
for name in tuple(sys.modules):
|
|
if name.split('.')[0] in binding.NUMERICAL_ROOTS:
|
|
monkeypatch.delitem(sys.modules, name)
|
|
numerical_inventory(tmp_path, monkeypatch)
|
|
dependency_path = tmp_path / 'replicator_dependency.py'
|
|
dependency_path.write_text('def seed(value):\n return value\n')
|
|
dependency = load('_replicator_dependency_test', dependency_path, monkeypatch)
|
|
module_path = tmp_path / 'replicator_core.py'
|
|
module_path.write_text('from _replicator_dependency_test import seed\n')
|
|
module = load(cli.OPTIONAL_REPLICATOR, module_path, monkeypatch)
|
|
monkeypatch.setattr(binding, '_sources', lambda *_: ({}, set()))
|
|
monkeypatch.setattr(binding, '_encode', lambda *_: None)
|
|
monkeypatch.setattr(binding, '_imports', lambda *_, **__: {})
|
|
report = cli.preimport_replicator()
|
|
assert report['available'] and report['missing_module'] is None
|
|
assert report['sampling_scope'] == 'no configured sampling' and not report['complete_rng_state_claim']
|
|
proposal = cli.proposed_inventory(binding, object())
|
|
assert dependency.__name__ not in proposal['modules']
|
|
for actual, path in ((module, module_path),):
|
|
assert proposal['modules'][actual.__name__] == {
|
|
'path': str(path), 'sha256': binding.sha256(path.read_bytes())}
|
|
assert binding._dependency_inventory(proposal, set(), set()) == proposal
|
|
cli.check_replicator_inventory(proposal)
|
|
|
|
|
|
def test_proposed_inventory_aggregates_independent_invalid_entries(tmp_path, monkeypatch):
|
|
from test_contact_cycle_sim_evaluation_binding import binding, load, numerical_inventory
|
|
|
|
for name in tuple(sys.modules):
|
|
if name.split('.')[0] in binding.NUMERICAL_ROOTS:
|
|
monkeypatch.delitem(sys.modules, name)
|
|
numerical_inventory(tmp_path, monkeypatch)
|
|
source = tmp_path / 'provider.py'
|
|
source.write_text('')
|
|
provider = load('diagnostic_provider', source, monkeypatch)
|
|
for name in ('broken_provider_one', 'broken_provider_two'):
|
|
module = cli.ModuleType(name)
|
|
monkeypatch.setitem(sys.modules, name, module)
|
|
setattr(provider, name, module)
|
|
monkeypatch.setattr(binding, '_sources', lambda *_: ({}, set()))
|
|
monkeypatch.setattr(binding, '_encode', lambda *_: None)
|
|
monkeypatch.setattr(binding, '_imports', lambda *_, **__: {
|
|
'diagnostic_provider': {'path': str(source), 'sha256': binding.sha256(source.read_bytes())}})
|
|
monkeypatch.setattr(binding, 'REQUIRED_RUNTIME', {**binding.REQUIRED_RUNTIME,
|
|
**{n: ('test-required', '/missing', '0' * 64)
|
|
for n in ('broken_provider_one', 'broken_provider_two')}})
|
|
(tmp_path / 'site/warp/bin/synthetic.so').unlink()
|
|
(tmp_path / 'site/torch/lib/synthetic.so').unlink()
|
|
monkeypatch.setattr(binding, '_dependency_inventory', lambda *_: pytest.fail('invalid proposal accepted'))
|
|
with pytest.raises(cli.InventoryPreparationError) as caught:
|
|
cli.proposed_inventory(binding, object())
|
|
errors = caught.value.report['invalid_entries']
|
|
assert {row['entry'] for row in errors} == {
|
|
'broken_provider_one', 'broken_provider_two', str(tmp_path / 'site/warp/bin'),
|
|
str(tmp_path / 'site/torch/lib')}
|
|
assert all('sha256' not in row for row in errors)
|
|
|
|
|
|
def test_prepare_inventory_diagnostic_is_fail_and_never_manifest(tmp_path, monkeypatch):
|
|
events = []
|
|
fake = NS(cfg=object(), binding=NS(FORMAT='test-v4', PROVENANCE_SCOPE='development', PROVENANCE_LIMITATIONS=(),
|
|
prepare_binding=lambda **_: pytest.fail('must not bind'),
|
|
module_diagnostic=lambda name: {'observed': name}),
|
|
arguments={}, app=NS(close=lambda **_: events.append('normal_app_close')))
|
|
monkeypatch.setattr(cli, 'real_runtime', lambda *_: fake)
|
|
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
|
|
diagnostics = [{'stage': 'dependency-module', 'entry': name, 'error': 'no real file'}
|
|
for name in ('one', 'two')]
|
|
|
|
def discover(*_):
|
|
raise cli.InventoryPreparationError(diagnostics)
|
|
|
|
monkeypatch.setattr(cli, 'proposed_inventory', discover)
|
|
args = NS(output=tmp_path / 'prepare', inventory=None, trial_index=0,
|
|
artifact_root=tmp_path, discover_inventory=True)
|
|
with pytest.raises(cli.InventoryPreparationError):
|
|
cli.prepare(args)
|
|
report = json.loads((args.output / 'inventory.diagnostic.json').read_bytes())
|
|
assert report['status'] == 'FAIL' and report['inventory_status'] == 'unreviewed_diagnostic'
|
|
assert not report['execution_authorized'] and report['invalid_entries'] == diagnostics
|
|
assert report['module_observations']['sys'] == {'observed': 'sys'}
|
|
assert sorted(p.name for p in args.output.iterdir()) == ['inventory.diagnostic.json', 'preparation-failure.json']
|
|
assert events == ['normal_app_close']
|
|
|
|
|
|
def test_constructor_failure_retains_actual_binding_diagnostic_without_preclose(lifecycle, tmp_path):
|
|
f = lifecycle
|
|
from test_contact_cycle_sim_evaluation_binding import binding
|
|
expected = binding.canonical_json({'effective_config': {'mutation': False}})
|
|
actual = binding.canonical_json({'effective_config': {'mutation': True}})
|
|
|
|
def constructor(**kwargs):
|
|
assert kwargs['cfg'] is f.cfg
|
|
f.cfg.mutation = True
|
|
raise RuntimeError('construction failed before environment returned')
|
|
|
|
def verify(row):
|
|
if f.cfg.mutation:
|
|
raise binding.BindingMismatchError(expected, actual)
|
|
|
|
f.api.environment = constructor
|
|
result = cli.execute_lifecycle(f.api, f.cfg, f.protocol, [0], tmp_path, f.budget, verify)
|
|
assert not result['continuation_eligible'] and f.cfg.mutation
|
|
assert result['failures'][0]['context']['phase'] == 'initialization'
|
|
diagnostic = result['failures'][-1]['context']['binding_diagnostic']
|
|
assert diagnostic['differences'][0]['path'] == '/effective_config/mutation'
|
|
assert diagnostic['differences'][0]['actual']['value'] is True
|
|
assert not (tmp_path / 'preclose-filesystem.json').exists()
|
|
|
|
|
|
def test_worker_persists_binding_diagnostic_before_fast_close_without_preclose(tmp_path, monkeypatch):
|
|
args, grant, _ = trusted_inputs(tmp_path)
|
|
row = reservation(clock=b.boottime())
|
|
row['grant'] = grant
|
|
output = args.artifact_root / 'launch-0'
|
|
output.mkdir()
|
|
b.durable_write(args.artifact_root / 'budget.json', b.encode({'launches': [row]}))
|
|
from test_contact_cycle_sim_evaluation_binding import binding
|
|
error = binding.BindingMismatchError(b'{"imports":{"x":1}}', b'{"imports":{"x":2}}')
|
|
closed = []
|
|
|
|
def verify(*_a, **_kw):
|
|
raise error
|
|
|
|
def close(*, exit_code):
|
|
saved = json.loads((output / 'result.json').read_bytes())
|
|
assert saved['failures'][-1]['context']['binding_diagnostic'] == error.binding_diagnostic
|
|
assert not saved['continuation_eligible'] and exit_code == 1
|
|
assert not (output / 'preclose-filesystem.json').exists()
|
|
closed.append(True)
|
|
|
|
runtime = NS(binding=NS(verify_binding=verify), arguments={'dependency_inventory': {}},
|
|
app=NS(close=close))
|
|
monkeypatch.setattr(cli, 'real_runtime', lambda *_: runtime)
|
|
monkeypatch.setattr(cli, 'verify_constructor_export', lambda _: None)
|
|
monkeypatch.setattr(cli, 'check_replicator_inventory', lambda *_: None)
|
|
payload = {'launch_identity': cli.launch_identity(), 'reservation': row, 'arguments': {
|
|
'artifact_root': str(args.artifact_root), 'inventory': str(args.inventory),
|
|
'manifest': str(args.manifest), 'trusted_manifest_sha256': args.trusted_manifest_sha256,
|
|
'protocol': args.protocol}}
|
|
assert cli.run_worker(payload) == 1 and closed == [True]
|
|
|
|
|
|
@pytest.mark.parametrize('drift', ['constructor_config', 'unexpected_module'])
|
|
def test_immediate_postconstructor_binding_stops_before_reset_or_inference(lifecycle, drift, tmp_path):
|
|
from test_contact_cycle_sim_evaluation_binding import binding
|
|
|
|
f = lifecycle
|
|
f.fault.phase = drift
|
|
diagnostic = {'difference_count': 1, 'differences': [{'path': '/imports/isaaclab.unexpected'}]}
|
|
|
|
def verify(row):
|
|
f.verify(row)
|
|
if drift == 'unexpected_module' and 'construct' in f.events:
|
|
error = ValueError('unexpected actual module')
|
|
error.binding_diagnostic = diagnostic
|
|
raise error
|
|
|
|
result = cli.execute_lifecycle(f.api, f.cfg, f.protocol, [0], tmp_path, f.budget, verify)
|
|
assert not result['continuation_eligible']
|
|
assert result['failures'][0]['causes'] == ['identity_external_failure']
|
|
assert result['observed_counts']['initialization_steps'] == 1 # constructor only
|
|
assert result['observed_counts']['rollout_steps'] == 0
|
|
assert not {'reset_0', 'inference', 'control_0', 'step_0', 'observer_enter'} & set(f.events)
|
|
assert f.events[f.events.index('construct') + 1] == 'verify_0'
|
|
assert f.events[-3:] == ['env_close', 'hooks_exit', 'verify_0']
|
|
assert 'durable_0' in f.events
|
|
if drift == 'unexpected_module':
|
|
assert result['failures'][0]['context']['binding_diagnostic'] == diagnostic
|
|
assert result['failures'][-1]['context']['binding_diagnostic'] == diagnostic
|
|
# The real comparator includes new imports, not just the lifecycle test double.
|
|
report = binding.binding_difference(b'{"imports":{}}', b'{"imports":{"isaaclab.unexpected":"new"}}')
|
|
assert report['difference_count'] == 1
|
|
|
|
|
|
def test_external_failure_bounded_traceback_excludes_locals_and_source():
|
|
def fail(depth):
|
|
secret = 'private-local-value-not-for-artifacts' # noqa: F841 - intentional private local
|
|
if depth:
|
|
return fail(depth - 1)
|
|
raise TypeError('cannot pickle module')
|
|
|
|
try:
|
|
fail(45)
|
|
except TypeError as error:
|
|
failure = cli.external_failure('initialization', error)
|
|
encoded = json.dumps(failure)
|
|
trace = failure['context']['traceback']
|
|
assert failure['fatal'] and failure['causes'] == ['infrastructure_external_failure']
|
|
assert len(trace['frames']) == 32 and trace['omitted_outer_frames'] > 0
|
|
assert trace['frames'][-1]['function'] == 'fail'
|
|
assert trace['frames'][-1]['file'] == __file__
|
|
assert all(set(frame) == {'file', 'line', 'function'} for frame in trace['frames'])
|
|
assert 'private-local-value' not in encoded and 'raise TypeError' not in encoded
|
|
assert cli.external_failure('step', RuntimeError('unraised'))['context']['traceback'] == {
|
|
'frames': [], 'omitted_outer_frames': 0}
|
|
|
|
|
|
def test_constructor_failure_traceback_persisted_without_continuation(lifecycle):
|
|
lifecycle.fault.phase = 'construct'
|
|
result = lifecycle.run([0, 1])
|
|
failure = result['failures'][0]
|
|
assert failure['context']['phase'] == 'initialization' and failure['fatal']
|
|
frames = failure['context']['traceback']['frames']
|
|
assert [f['function'] for f in frames] == ['execute_lifecycle', '__init__']
|
|
persisted = json.loads((lifecycle.state.recorders[0].path / 'durable.json').read_bytes())
|
|
assert persisted['external'] == failure
|
|
assert not result['continuation_eligible'] and not result['accepted_skill']
|
|
assert 'reset_0' not in lifecycle.events and 'step_0' not in lifecycle.events
|