9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
438 lines
25 KiB
Python
438 lines
25 KiB
Python
"""CPU-only fixed-profile/adoption contracts; never launches Kit or creates real grants."""
|
|
import gzip
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
spec = importlib.util.spec_from_file_location('p2_extended_runner', ROOT / 'scripts/evaluate_cycle_sim_residual.py')
|
|
r = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(r)
|
|
b = r.cpu_module('evaluation_budget')
|
|
a = r.cpu_module('evaluation_adoption')
|
|
p = r.cpu_module('evaluation_protocol')
|
|
|
|
|
|
def put(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(b.encode(value))
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def no_launch(monkeypatch):
|
|
monkeypatch.setattr(r.subprocess, 'Popen', lambda *_a, **_k: pytest.fail('CPU only'))
|
|
monkeypatch.setattr(r.os, 'pidfd_open', lambda *_: pytest.fail('CPU only'), raising=False)
|
|
monkeypatch.setattr(r.signal, 'pidfd_send_signal', lambda *_: pytest.fail('CPU only'), raising=False)
|
|
|
|
|
|
def test_only_two_wall_values_and_version_metadata_differ():
|
|
old = json.loads((ROOT / r.PROTOCOLS[1]).read_bytes())
|
|
new = json.loads((ROOT / r.PROTOCOLS[2]).read_bytes())
|
|
assert new.pop('resource_profile') == b.EXTENDED_PROFILE
|
|
assert new.pop('format') == 'skillfactory.contact-cycle-evaluation-protocol.v2'
|
|
new['format'] = old['format']
|
|
assert new['resources'].pop('P2_launch_wall_cap_s') == 10800
|
|
new['resources']['P2_launch_wall_cap_s'] = old['resources']['P2_launch_wall_cap_s']
|
|
assert new['resources'].pop('aggregate_same_boot_window_s') == 14400
|
|
new['resources']['aggregate_same_boot_window_s'] = old['resources']['aggregate_same_boot_window_s']
|
|
assert new == old
|
|
assert b.digest(ROOT / r.PROTOCOLS[0]) == '8ad69f1c3a0e1a85b20050088574a50c4d90dc88ec85e14e66826bae1e7823c3'
|
|
protocol = p.load_protocol(ROOT / r.PROTOCOLS[2])
|
|
assert [t.index for t in protocol.trials] == list(range(20))
|
|
assert protocol.trials == p.load_protocol(ROOT / r.PROTOCOLS[1]).trials
|
|
assert r.protocol_stage(r.PROTOCOLS[2]) == 'P2'
|
|
for change in ('wall', 'profile', 'stage', 'gate'):
|
|
document = json.loads((ROOT / r.PROTOCOLS[2]).read_bytes())
|
|
if change == 'wall':
|
|
document['resources']['P2_launch_wall_cap_s'] += 1
|
|
elif change == 'profile':
|
|
document['resource_profile'] += '-arbitrary'
|
|
elif change == 'stage':
|
|
document['stage'] = 'P1'
|
|
else:
|
|
document['thresholds']['functional']['root_tilt_rad_max'] += .01
|
|
with pytest.raises(ValueError):
|
|
p.Protocol(document)
|
|
|
|
|
|
@pytest.fixture
|
|
def source(tmp_path, monkeypatch):
|
|
source, target = tmp_path / 'p1-03', tmp_path / 'p2-01'
|
|
source.mkdir(); target.mkdir()
|
|
monkeypatch.setattr(a, 'SOURCE_ROOT', source)
|
|
monkeypatch.setattr(a, 'TARGET_ROOT', target)
|
|
monkeypatch.setattr(b, 'EXTENDED_ROOT', target)
|
|
monkeypatch.setattr(b, 'boot_id', lambda: 'boot')
|
|
monkeypatch.setattr(b, 'boottime', lambda: 100.)
|
|
frozen = tmp_path / 'frozen/candidate'
|
|
put(frozen, {'unchanged': True})
|
|
sha = b.digest(frozen)
|
|
monkeypatch.setattr(a, 'CANDIDATE_SHA256', sha)
|
|
protocol = p.load_protocol(ROOT / r.PROTOCOLS[0])
|
|
trials = [r.asdict(t) for t in protocol.trials]
|
|
rows = []
|
|
for i, tick in enumerate((1829, 1770)):
|
|
launch = source / f'launch-{i}'
|
|
metrics = {'trial': trials[i], 'accepted_skill': False, 'functional_passed': False,
|
|
'protocol_complete': False, 'completed_clock_cycles': 0, 'passed_prefix_cycles': 0,
|
|
'observed_physics_ticks': tick, 'all_causes': ['P1_root_horizontal'],
|
|
'first_failure': {'global_tick': tick, 'time_s': tick * .002, 'cycle_index': 0,
|
|
'local_tick': tick, 'kind': 'tick', 'causes': ['P1_root_horizontal'], 'phase': 1}}
|
|
manifest = {'protocol': {'path': str(ROOT / r.PROTOCOLS[0]), 'sha256': b.digest(ROOT / r.PROTOCOLS[0]),
|
|
'trials': trials}, 'trial_seeds': {'0': 93501, '1': 93501},
|
|
'frozen_policy': {'candidate_path': str(frozen), 'candidate_sha256': sha},
|
|
'effective_config': {'seed': 'protocol.trial.seed', 'controller': 'original'},
|
|
'sources': {'evaluation': [{'path': 'historical-source', 'sha256': 'a' * 64}]}}
|
|
put(launch / 'manifest.json', manifest)
|
|
put(launch / 'inventory.json', {'original': True})
|
|
put(launch / 'launch-identity.json', {'original': True})
|
|
grant = {'stage': 'P1', 'trial_indices': [i], 'launch_index': i, 'campaign_id': a.SOURCE_CAMPAIGN,
|
|
'artifact_root': str(source), 'boot_id': 'boot', 'protocol_sha256': b.digest(ROOT / r.PROTOCOLS[0]),
|
|
**{key: b.digest(launch / name) for key, name in (
|
|
('manifest_sha256', 'manifest.json'), ('inventory_sha256', 'inventory.json'),
|
|
('launch_identity_sha256', 'launch-identity.json'))}}
|
|
put(launch / 'grant.json', grant)
|
|
counts = dict(attempted_steps=tick, completed_steps=tick, initialization_steps=0, rollout_steps=tick)
|
|
row = dict(index=i, status='closed', cleanup_clean=True, continuation_eligible=True,
|
|
reserved_physical_calls=14500, rollout_cap=13500, initialization_cap=1000,
|
|
grant=grant, observed_counts=counts, ended_boottime=20 + i)
|
|
rows.append(row)
|
|
put(launch / 'reservation.json', {k: v for k, v in row.items() if k not in ('cleanup_clean', 'ended_boottime')}
|
|
| {'status': 'reserved', 'continuation_eligible': False, 'observed_counts': None})
|
|
result_rows = [{'status': 'not_run'}, {'status': 'not_run'}]
|
|
result_rows[i] = metrics
|
|
put(launch / 'result.json', {'accepted_skill': False, 'optimizer_steps': 0, 'selected_rows': [i],
|
|
'rows': result_rows, 'failures': [], 'preclose_identity_verified': True,
|
|
'continuation_eligible': True, 'observed_counts': counts})
|
|
put(launch / f'trial-{i:02d}/metrics.json', metrics)
|
|
put(launch / 'supervisor.json', dict(observed_counts=counts, cleanup_clean=True, continuation_eligible=True,
|
|
ownership_unknown=False, returncode=0, errors=[], stdout_discarded_bytes=0, stdout_eof_observed=True))
|
|
put(launch / 'preclose-filesystem.json', {'files': {str(frozen): sha}, 'frozen_root': str(frozen.parent)})
|
|
put(launch / 'postexit-filesystem.json', {'status': 'PASS', 'files_rehashed': 1})
|
|
with gzip.open(launch / f'trial-{i:02d}/trace.v1.jsonl.gz', 'wb') as stream:
|
|
stream.write(b.encode({'kind': 'initial', 'event': {'trial_index': i, 'clock': {'global_tick': 0}}}))
|
|
for step in range(1, tick + 1):
|
|
stream.write(b.encode({'kind': 'tick', 'event': {'trial_index': i, 'clock': {'global_tick': step}}}))
|
|
stream.write(b.encode({'kind': 'terminal', 'runtime_stop': {'trial_index': i, 'reason': 'failure',
|
|
'fatal': False, 'physics_ticks': tick, 'clock': {'global_tick': tick}, 'causes': ['P1_root_horizontal']},
|
|
'recording_faults': [], 'external_failure': None}))
|
|
ledger = {'format': b.FORMAT, 'campaign_id': a.SOURCE_CAMPAIGN, 'boot_id': 'boot',
|
|
'first_run_boottime': 10, 'launches': rows}
|
|
put(source / 'budget.json', ledger)
|
|
put(source / 'row1-preflight.json', {'cumulative_prior_consumed_physical_calls': 58000,
|
|
'budget_before': {**ledger, 'launches': rows[:1]}})
|
|
for name in ('FINAL_P1_ROW0.md', 'FINAL_P1_ROW1.md', 'PAIRED_P1_SUMMARY.md'):
|
|
(source / name).write_text('Both original P1_root_horizontal failures; not a pass.\n')
|
|
return NS(root=source, target=target, ledger=ledger)
|
|
|
|
|
|
def adopt(source):
|
|
return a.create_adoption(parent_review='CPU fixture explicit evidence review, not physical pass',
|
|
trusted_source_budget_sha256=b.digest(source.root / 'budget.json'))
|
|
|
|
|
|
def test_adoption_is_separate_read_only_no_debit_no_clock(source):
|
|
before = b.tree_digest(source.root)
|
|
digest = adopt(source)
|
|
doc = a.verify_adoption(source.target, digest)
|
|
assert b.tree_digest(source.root) == before
|
|
assert doc['historical_reserved_calls_before_reference'] == 58000
|
|
assert doc['historical_reserved_calls'] == 72500 and doc['local_reserved_calls_at_adoption'] == 0
|
|
assert doc['cumulative_reserved_calls_if_launched'] == 163500
|
|
assert [row['first_failure']['global_tick'] for row in doc['rows']] == [1829, 1770]
|
|
assert all(not row['functional_passed'] and not row['accepted_skill'] for row in doc['rows'])
|
|
assert not (source.target / 'budget.json').exists()
|
|
assert not list(source.target.glob('launch-*'))
|
|
with pytest.raises(FileExistsError):
|
|
adopt(source)
|
|
with pytest.raises(ValueError, match='parent_review'):
|
|
a.create_adoption(parent_review='', trusted_source_budget_sha256=b.digest(source.root / 'budget.json'))
|
|
with pytest.raises(ValueError, match='digest'):
|
|
a.verify_adoption(source.target, '0' * 64)
|
|
|
|
|
|
@pytest.mark.parametrize('mutation', ['failure', 'seed', 'partial', 'cleanup', 'accounting', 'trace', 'missing'])
|
|
def test_adoption_rejects_partial_forged_crossseed_unclean_and_mutation(source, mutation):
|
|
digest = adopt(source)
|
|
path = source.root / 'launch-1/trial-01/metrics.json'
|
|
value = json.loads(path.read_bytes())
|
|
if mutation == 'failure':
|
|
value['functional_passed'] = True
|
|
elif mutation == 'seed':
|
|
value['trial']['seed'] += 1
|
|
elif mutation == 'partial':
|
|
value['first_failure'] = None
|
|
elif mutation == 'cleanup':
|
|
path = source.root / 'launch-1/supervisor.json'; value = json.loads(path.read_bytes())
|
|
value['cleanup_clean'] = False
|
|
elif mutation == 'accounting':
|
|
path = source.root / 'row1-preflight.json'; value = json.loads(path.read_bytes())
|
|
value['cumulative_prior_consumed_physical_calls'] = 43500
|
|
elif mutation == 'trace':
|
|
path = source.root / 'launch-1/trial-01/trace.v1.jsonl.gz'
|
|
path.write_bytes(path.read_bytes()[:-10])
|
|
else:
|
|
path.unlink()
|
|
if mutation not in ('trace', 'missing'):
|
|
put(path, value)
|
|
with pytest.raises((ValueError, FileNotFoundError)):
|
|
a.verify_adoption(source.target, digest)
|
|
|
|
|
|
def test_one_future_reservation_separate_history_and_cross_profile_rejected(source):
|
|
sha = adopt(source)
|
|
grant = {'format': b.EXTENDED_GRANT_FORMAT, 'resource_profile': b.EXTENDED_PROFILE, 'stage': 'P2',
|
|
'trial_indices': list(range(20)), 'launch_index': 0, 'not_after_boottime': 500,
|
|
'p1_adoption_sha256': sha, 'boot_id': 'boot'}
|
|
kwargs = dict(clock=lambda: 100., boot=lambda: 'boot')
|
|
ledger = b.BudgetLedger(source.target, 'new-P2', resource_profile=b.EXTENDED_PROFILE, **kwargs)
|
|
assert ledger.data['first_run_boottime'] is None and ledger.data['launches'] == []
|
|
assert ledger.data['cumulative_reserved_calls'] == 72500
|
|
row = ledger.reserve(grant)
|
|
assert row['reserved_physical_calls'] == 91000 and row['rollout_cap'] == 90000
|
|
assert row['initialization_cap'] == 1000 and row['deadline_boottime'] == 10900
|
|
assert ledger.data['first_run_boottime'] == 100 and ledger.data['cumulative_reserved_calls'] == 163500
|
|
ledger.finish(counts={'completed_steps': 0}, clean=True, eligible=True)
|
|
with pytest.raises(RuntimeError, match='retry'):
|
|
ledger.reserve({**grant, 'launch_index': 1})
|
|
ledger.close()
|
|
with pytest.raises(RuntimeError, match='mismatch'):
|
|
b.BudgetLedger(source.target, 'new-P2', **kwargs)
|
|
legacy = source.target / 'legacy'; legacy.mkdir()
|
|
ledger = b.BudgetLedger(legacy, 'legacy', **kwargs)
|
|
with pytest.raises(RuntimeError, match='profile_mismatch'):
|
|
ledger.reserve(grant)
|
|
ledger.close()
|
|
(source.target / 'budget.json').write_bytes((legacy / 'budget.json').read_bytes())
|
|
with pytest.raises(RuntimeError, match='mismatch'):
|
|
b.BudgetLedger(source.target, 'legacy', resource_profile=b.EXTENDED_PROFILE, **kwargs)
|
|
|
|
|
|
@pytest.mark.parametrize('mutate_prior', [False, True])
|
|
def test_second_campaign_retains_failed_attempt_debit(source, monkeypatch, mutate_prior):
|
|
prior = source.target
|
|
documents = {
|
|
'budget.json': {'boot_id': 'boot', 'launches': [{'status': 'closed'}],
|
|
'historical_reserved_calls': 72500, 'local_reserved_calls': 91000,
|
|
'cumulative_reserved_calls': 163500},
|
|
'launch-0/result.json': {'continuation_eligible': False, 'accepted_skill': False,
|
|
'observed_counts': None,
|
|
'rows': [{'status': 'not_run', 'trial_index': i} for i in range(20)],
|
|
'failures': [{'message': 'retained bootstrap failure'}]},
|
|
'launch-0/supervisor.json': {'cleanup_clean': True, 'ownership_unknown': False, 'returncode': 1},
|
|
'launch-0/grant.json': {'consumed': True},
|
|
'launch-0/reservation.json': {'reserved_physical_calls': 91000},
|
|
}
|
|
for name, document in documents.items():
|
|
put(prior / name, document)
|
|
monkeypatch.setattr(a, 'PRIOR_P2_ROOT', prior)
|
|
pins = {name: b.digest(prior / name) for name in documents}
|
|
monkeypatch.setattr(a, 'PRIOR_P2_FILES', pins)
|
|
second = prior.parent / 'p2-02'; second.mkdir()
|
|
monkeypatch.setattr(a, 'SECOND_EXTENDED_ROOT', second)
|
|
monkeypatch.setattr(b, 'SECOND_EXTENDED_ROOT', second)
|
|
before = b.tree_digest(prior)
|
|
sha = a.create_adoption(parent_review='new CPU proposal, no run authority',
|
|
trusted_source_budget_sha256=b.digest(source.root / 'budget.json'),
|
|
target_root=second)
|
|
assert b.tree_digest(prior) == before
|
|
assert not (second / 'budget.json').exists()
|
|
doc = a.verify_adoption(second, sha)
|
|
assert doc['historical_reserved_calls'] == 163500
|
|
assert doc['cumulative_reserved_calls_if_launched'] == 254500
|
|
assert doc['prior_failed_P2']['observed_counts'] is None
|
|
grant = {'format': b.EXTENDED_GRANT_FORMAT, 'resource_profile': b.EXTENDED_PROFILE,
|
|
'stage': 'P2', 'trial_indices': list(range(20)), 'launch_index': 0,
|
|
'not_after_boottime': 500, 'p1_adoption_sha256': sha, 'boot_id': 'boot'}
|
|
ledger = b.BudgetLedger(second, 'second', resource_profile=b.EXTENDED_PROFILE,
|
|
clock=lambda: 100., boot=lambda: 'boot')
|
|
try:
|
|
if mutate_prior:
|
|
(prior / 'launch-0/result.json').write_text('{}')
|
|
with pytest.raises(ValueError, match='prior_P2_changed'):
|
|
ledger.reserve(grant)
|
|
assert ledger.data['launches'] == []
|
|
assert ledger.data['local_reserved_calls'] == 0
|
|
else:
|
|
ledger.reserve(grant)
|
|
assert ledger.data['cumulative_reserved_calls'] == 254500
|
|
assert ledger.data['historical_reserved_calls'] == 163500
|
|
ledger.finish(counts=None, clean=True, eligible=False)
|
|
with pytest.raises(RuntimeError, match='retry'):
|
|
ledger.reserve(grant)
|
|
assert b.tree_digest(prior) == before
|
|
finally:
|
|
ledger.close()
|
|
with pytest.raises(ValueError, match='fixed_root'):
|
|
b.extended_historical_calls(second.parent / 'p2-03')
|
|
|
|
|
|
def test_second_root_preparation_reaches_policy_without_launch(tmp_path, monkeypatch):
|
|
spec = importlib.util.spec_from_file_location('second_root_prep_test', ROOT / 'scripts/prepare_cycle_sim_binding.py')
|
|
prep = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(prep)
|
|
second = tmp_path / 'p2-02'; second.mkdir()
|
|
monkeypatch.setattr(prep.b, 'SECOND_EXTENDED_ROOT', second)
|
|
def boundary(*_):
|
|
raise RuntimeError('CPU policy boundary; no Kit launch')
|
|
monkeypatch.setattr(prep.r, 'storage_policy_check', boundary)
|
|
with pytest.raises(RuntimeError, match='CPU policy boundary'):
|
|
prep.supervise(second, tmp_path / 'unused-policy', 'unused', r.PROTOCOLS[2])
|
|
assert list(second.iterdir()) == []
|
|
with pytest.raises(ValueError, match='fixed_root'):
|
|
prep.supervise(tmp_path / 'p2-03', tmp_path / 'unused-policy', 'unused', r.PROTOCOLS[2])
|
|
|
|
|
|
def policy_fixture(source, monkeypatch):
|
|
monkeypatch.setattr(r, 'read_trusted', lambda path, _: path.read_bytes() if path.name == 'policy.json' else b'')
|
|
monkeypatch.setattr(r, 'mount_identity', lambda _: 'mount')
|
|
root = source.target
|
|
policy = {'format': b.EXTENDED_POLICY_FORMAT, 'artifact_root': str(root), 'limit_bytes': b.CAP_BYTES,
|
|
'scope': b.STORAGE_SCOPE, 'approval_sha256': r.STORAGE_APPROVAL_SHA256,
|
|
'native_paths': r.native_paths(root), 'check_interval_seconds': 1, 'max_native_growth_bytes': b.CAP_BYTES,
|
|
'min_free_bytes': 2 * b.CAP_BYTES, 'coverage_gaps': list(r.NATIVE_GAPS), 'native_hard_cap': False,
|
|
'boot_id': 'boot', 'mount_identity': 'mount', 'not_after_boottime': 14500., 'issued_boottime': 100.,
|
|
'resource_profile': b.EXTENDED_PROFILE, 'shared_native_paths': r.shared_native_paths(),
|
|
'task_owned_cache_paths': r.task_owned_cache_paths(root), 'free_space_paths': r.free_space_paths(root),
|
|
'free_space_mount_identities': {p: 'mount' for p in r.free_space_paths(root)}}
|
|
args = NS(artifact_root=root, protocol=r.PROTOCOLS[2], storage_policy=root / 'policy.json',
|
|
trusted_storage_policy_sha256='0' * 64)
|
|
return args, policy
|
|
|
|
|
|
def test_extended_policy_expiry_boundary_legacy_unchanged(source, monkeypatch):
|
|
args, policy = policy_fixture(source, monkeypatch)
|
|
put(args.storage_policy, policy)
|
|
assert r.storage_policy_check(args, b) == policy
|
|
for change in ({'not_after_boottime': 14500.001}, {'issued_boottime': 101}, {'boot_id': 'other'},
|
|
{'format': 'skillfactory.output-storage-policy.v4'}):
|
|
put(args.storage_policy, {**policy, **change})
|
|
with pytest.raises(RuntimeError):
|
|
r.storage_policy_check(args, b)
|
|
old = {k: v for k, v in policy.items() if k not in ('issued_boottime', 'resource_profile')}
|
|
old.update(format='skillfactory.output-storage-policy.v4', not_after_boottime=3700)
|
|
args.protocol = r.PROTOCOLS[0]
|
|
put(args.storage_policy, old)
|
|
assert r.storage_policy_check(args, b) == old
|
|
old['not_after_boottime'] += .001
|
|
put(args.storage_policy, old)
|
|
with pytest.raises(RuntimeError):
|
|
r.storage_policy_check(args, b)
|
|
|
|
|
|
def test_grant_admission_stays_short_and_binds_twenty_rows(source, monkeypatch):
|
|
adoption = adopt(source)
|
|
monkeypatch.setattr(r, 'launch_identity', lambda: {'CPU': True})
|
|
args = NS(protocol=r.PROTOCOLS[2], artifact_root=source.target, manifest=source.target / 'binding.json',
|
|
inventory=source.target / 'inventory.json', trusted_storage_policy_sha256='e' * 64)
|
|
put(args.manifest, {'CPU': True}); put(args.inventory, {'CPU': True})
|
|
args.trusted_manifest_sha256 = b.digest(args.manifest)
|
|
grant = dict(format=b.EXTENDED_GRANT_FORMAT, resource_profile=b.EXTENDED_PROFILE, campaign_id='P2',
|
|
launch_index=0, stage='P2', trial_indices=list(range(20)), manifest_sha256=args.trusted_manifest_sha256,
|
|
protocol_sha256=b.digest(ROOT / args.protocol), inventory_sha256=b.digest(args.inventory),
|
|
artifact_root=str(args.artifact_root), boot_id='boot', not_after_boottime=3700, parent_review='CPU review',
|
|
p1_adoption_sha256=adoption, storage_policy_sha256=args.trusted_storage_policy_sha256,
|
|
storage_scope=b.STORAGE_SCOPE, launch_identity_sha256=r.hashlib.sha256(b.encode(r.launch_identity())).hexdigest())
|
|
assert r.grant_check(grant, args, b) == grant
|
|
for change in ({'not_after_boottime': 3700.001}, {'trial_indices': list(range(19))}, {'launch_index': 2},
|
|
{'stage': 'P1'}, {'resource_profile': 'arbitrary'}, {'p1_adoption_sha256': '0' * 64}):
|
|
with pytest.raises(ValueError):
|
|
r.grant_check({**grant, **change}, args, b)
|
|
|
|
|
|
@pytest.mark.parametrize('mutate_after_exit', [False, True])
|
|
def test_supervisor_rechecks_external_adoption_after_exit_and_always_cleans(source, monkeypatch, mutate_after_exit):
|
|
sha = adopt(source)
|
|
root = source.target
|
|
args = NS(protocol=r.PROTOCOLS[2], artifact_root=root, manifest=root / 'binding.json',
|
|
inventory=root / 'inventory.json', grant=root / 'grant.json', device='cpu')
|
|
put(args.manifest, {'CPU': True}); put(args.inventory, {'CPU': True})
|
|
args.trusted_manifest_sha256 = b.digest(args.manifest)
|
|
grant = {'format': b.EXTENDED_GRANT_FORMAT, 'resource_profile': b.EXTENDED_PROFILE, 'stage': 'P2',
|
|
'trial_indices': list(range(20)), 'launch_index': 0, 'not_after_boottime': 500,
|
|
'p1_adoption_sha256': sha, 'boot_id': 'boot', 'campaign_id': 'CPU-P2'}
|
|
put(args.grant, grant); args.trusted_grant_sha256 = b.digest(args.grant)
|
|
monkeypatch.setattr(r, 'grant_check', lambda value, *_: value) # exact schema tested separately
|
|
monkeypatch.setattr(r, 'storage_policy_check', lambda *_: {'not_after_boottime': 14000, 'limit_bytes': b.CAP_BYTES})
|
|
monkeypatch.setattr(r, 'launch_identity', lambda: {'CPU': True})
|
|
monkeypatch.setattr(r, 'launch_command', lambda *_: ['CPU-no-process'])
|
|
monkeypatch.setattr(r, 'NativeMonitor', lambda *_: NS(check=lambda: None, report=lambda: {'CPU': True}))
|
|
monkeypatch.setattr(r, 'OwnedProcesses', lambda: NS(
|
|
supervisor=(1, 2), known={(3, 4)}, unknown=False, attach=lambda _: None, discover=lambda: {}))
|
|
monkeypatch.setattr(r.ctypes, 'CDLL', lambda *_a, **_k: NS(prctl=lambda *_: 0))
|
|
monkeypatch.setattr(r.socket, 'socketpair', lambda: (
|
|
NS(close=lambda: None), NS(close=lambda: None, set_inheritable=lambda _: None, fileno=lambda: 99)))
|
|
ledger_class, budget_class = b.BudgetLedger, b.LaunchBudget
|
|
monkeypatch.setattr(b, 'BudgetLedger', lambda *args, **kwargs: ledger_class(
|
|
*args, clock=lambda: 100., boot=lambda: 'boot', **kwargs))
|
|
monkeypatch.setattr(b, 'LaunchBudget', lambda *args, **kwargs: budget_class(
|
|
*args, clock=lambda: 100., boot=lambda: 'boot', **kwargs))
|
|
events = []
|
|
|
|
def spawn(*_a, **_k):
|
|
events.append('spawn')
|
|
put(root / 'launch-0/result.json', {'observed_counts': {'completed_steps': 3},
|
|
'continuation_eligible': True, 'preclose_identity_verified': True})
|
|
put(root / 'launch-0/preclose-filesystem.json', {'CPU': True})
|
|
return NS(pid=3, stdout=None, poll=lambda: 0, returncode=0, wait=lambda **_: None)
|
|
monkeypatch.setattr(r.subprocess, 'Popen', spawn)
|
|
monkeypatch.setattr(r, 'exchange_worker', lambda _child, _channel, payload, *_:
|
|
events.append(payload['reservation']['grant']['trial_indices']))
|
|
monkeypatch.setattr(r, 'StdoutCapture', lambda *_: NS(
|
|
eof=True, discarded=0, error=None, close=lambda: None, check=lambda: None))
|
|
monkeypatch.setattr(r, 'verify_filesystem_inputs', lambda _: {'status': 'PASS'})
|
|
|
|
def cleanup(*_):
|
|
events.append('cleanup')
|
|
if mutate_after_exit:
|
|
path = source.root / 'launch-0/stdout.log'
|
|
path.write_text('changed after exit')
|
|
return True
|
|
monkeypatch.setattr(r, 'cleanup_group', cleanup)
|
|
assert r.supervise(args) == (1 if mutate_after_exit else 0)
|
|
assert events == ['spawn', list(range(20)), 'cleanup']
|
|
ledger = json.loads((root / 'budget.json').read_bytes())
|
|
assert len(ledger['launches']) == 1 and ledger['cumulative_reserved_calls'] == 163500
|
|
row = ledger['launches'][0]
|
|
assert row['status'] == 'closed' and row['cleanup_clean']
|
|
assert row['continuation_eligible'] is (not mutate_after_exit)
|
|
adopted = json.loads((root / 'launch-0/adopted-p1-evidence.json').read_bytes())
|
|
assert [row['first_failure']['global_tick'] for row in adopted['rows']] == [1829, 1770]
|
|
assert (root / 'launch-0/postexit-adoption.json').exists() is (not mutate_after_exit)
|
|
if mutate_after_exit:
|
|
assert json.loads((root / 'launch-0/supervisor.json').read_bytes())['errors']
|
|
|
|
|
|
def test_extended_storage_policy_requires_slot_plus_cleanup_coverage(source, monkeypatch):
|
|
monkeypatch.setattr(r, 'storage_policy_check', lambda *_: {'not_after_boottime': 11019.999})
|
|
monkeypatch.setattr(r, 'read_trusted', lambda *_: b.encode({}))
|
|
monkeypatch.setattr(r, 'grant_check', lambda *_: {'launch_index': 0, 'not_after_boottime': 200})
|
|
args = NS(protocol=r.PROTOCOLS[2], grant=source.target / 'never', trusted_grant_sha256='0' * 64)
|
|
with pytest.raises(RuntimeError, match='storage_policy_expires'):
|
|
r.supervise(args)
|
|
assert not (source.target / 'budget.json').exists()
|
|
|
|
|
|
@pytest.mark.parametrize('mutation', ['missing_initial', 'missing_terminal', 'wrong_terminal_trial', 'tick_gap'])
|
|
def test_adoption_construction_rejects_partial_or_cross_trial_trace(source, mutation):
|
|
path = source.root / 'launch-1/trial-01/trace.v1.jsonl.gz'
|
|
with gzip.open(path, 'rt') as stream:
|
|
records = [json.loads(line) for line in stream]
|
|
if mutation == 'missing_initial':
|
|
records.pop(0)
|
|
elif mutation == 'missing_terminal':
|
|
records.pop()
|
|
elif mutation == 'wrong_terminal_trial':
|
|
records[-1]['runtime_stop']['trial_index'] = 0
|
|
else:
|
|
records[10]['event']['clock']['global_tick'] += 1
|
|
with gzip.open(path, 'wb') as stream:
|
|
for record in records:
|
|
stream.write(b.encode(record))
|
|
with pytest.raises(ValueError, match='trace_'):
|
|
adopt(source)
|
|
assert not (source.target / 'p1-adoption.json').exists()
|