9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
563 lines
28 KiB
Python
563 lines
28 KiB
Python
"""CPU-only launch contract/lifecycle mocks. NEVER imports Kit or starts a worker."""
|
|
import contextlib
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
import rootxy_launch_v2 as a
|
|
|
|
spec = importlib.util.spec_from_file_location('rootxy_runtime_cpu_test', ROOT /
|
|
'source/SkillFactory/SkillFactory/legs/contact_cycle_sim/rootxy_runtime_v2.py')
|
|
runtime = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(runtime)
|
|
|
|
|
|
@pytest.fixture
|
|
def desktop_gpu(tmp_path, monkeypatch):
|
|
library = tmp_path / 'nvml.so'; library.write_bytes(b'mock')
|
|
client = {'pid': 7, 'starttime': 123, 'sha256': 'executable-hash'}
|
|
identity = {'library': str(library), 'sha256': a.b.digest(library), 'uuid': 'GPU-test',
|
|
'index': 0, 'desktop_clients': [client]}
|
|
current = {'compute': set(), 'graphics': {7}}
|
|
monkeypatch.setattr(a.ctypes, 'CDLL', lambda _: object())
|
|
def call(self, name, *args):
|
|
if name == 'nvmlDeviceGetUUID':
|
|
args[1].value = b'GPU-test'
|
|
monkeypatch.setattr(a.GPUOwnership, 'call', call)
|
|
monkeypatch.setattr(a.GPUOwnership, 'clients', lambda _: current)
|
|
monkeypatch.setattr(a, 'desktop_identity', lambda pid, hash_executable=True:
|
|
dict(client) if hash_executable else {k: v for k, v in client.items() if k != 'sha256'})
|
|
return SimpleNamespace(identity=identity, client=client, current=current,
|
|
owned=SimpleNamespace(discover=lambda: {99: object()}))
|
|
|
|
|
|
@pytest.mark.parametrize('compute,graphics,final,passes', [
|
|
(set(), {7}, False, True), ({99}, {7, 99}, False, True),
|
|
(set(), set(), True, True), (set(), {7}, True, True),
|
|
({7}, {7}, False, False), ({8}, {7}, False, False),
|
|
(set(), {7, 8}, False, False), ({99}, {7}, True, False),
|
|
(set(), {7, 99}, True, False),
|
|
])
|
|
def test_gpu_desktop_sharing_never_allows_foreign_compute_or_new_graphics(desktop_gpu, compute, graphics, final, passes):
|
|
f = desktop_gpu
|
|
gpu = a.GPUOwnership(f.identity)
|
|
f.current.update(compute=compute, graphics=graphics)
|
|
if passes:
|
|
report = gpu.check(f.owned, final=final)
|
|
assert report['approved_desktop_graphics_pids'] == sorted(graphics & {7})
|
|
assert report['clean'] == (not compute and not (graphics - {7}))
|
|
else:
|
|
with pytest.raises(RuntimeError, match='GPU ownership'):
|
|
gpu.check(f.owned, final=final)
|
|
|
|
|
|
@pytest.mark.parametrize('change', ['foreign_compute', 'new_graphics', 'missing_graphics', 'birth', 'hash', 'duplicate'])
|
|
def test_gpu_admission_requires_exact_parent_desktop_baseline(desktop_gpu, change):
|
|
f = desktop_gpu
|
|
if change == 'foreign_compute': f.current['compute'] = {7}
|
|
if change == 'new_graphics': f.current['graphics'].add(8)
|
|
if change == 'missing_graphics': f.current['graphics'].clear()
|
|
if change in ('birth', 'hash'):
|
|
f.identity['desktop_clients'] = [{**f.client, ('starttime' if change == 'birth' else 'sha256'): 'changed'}]
|
|
if change == 'duplicate': f.identity['desktop_clients'] *= 2
|
|
with pytest.raises((ValueError, RuntimeError)):
|
|
a.GPUOwnership(f.identity)
|
|
|
|
|
|
def test_gpu_runtime_pid_reuse_and_final_executable_change_fail(desktop_gpu, monkeypatch):
|
|
f = desktop_gpu
|
|
gpu = a.GPUOwnership(f.identity)
|
|
monkeypatch.setattr(a, 'desktop_identity', lambda *args, **kwargs: {'pid': 7, 'starttime': 999})
|
|
with pytest.raises(RuntimeError, match='identity changed'):
|
|
gpu.check(f.owned)
|
|
with pytest.raises(RuntimeError, match='identity changed'):
|
|
gpu.check(f.owned, final=True)
|
|
|
|
|
|
@pytest.mark.parametrize('uid,command,comm,allowed', [
|
|
(0, b'/usr/lib/xorg/Xorg\0vt2\0', 'Xorg', True),
|
|
(1000, b'/usr/lib/xorg/Xorg\0', 'Xorg', False),
|
|
(0, b'/usr/bin/other\0', 'Xorg', False),
|
|
(0, b'/usr/lib/xorg/Xorg\0', 'other', False),
|
|
])
|
|
def test_inaccessible_executable_only_explicit_root_xorg_metadata(tmp_path, monkeypatch, uid, command, comm, allowed):
|
|
proc = tmp_path / '7'; proc.mkdir()
|
|
fields = ['0'] * 20; fields[19] = '123'
|
|
(proc / 'stat').write_text('7 (Xorg) ' + ' '.join(fields))
|
|
(proc / 'cmdline').write_bytes(command)
|
|
(proc / 'comm').write_text(comm)
|
|
old_stat, old_resolve = Path.stat, Path.resolve
|
|
monkeypatch.setattr(Path, 'stat', lambda self, *args, **kwargs:
|
|
SimpleNamespace(st_uid=uid) if self == proc else old_stat(self, *args, **kwargs))
|
|
def resolve(self, *args, **kwargs):
|
|
if self == proc / 'exe':
|
|
raise PermissionError('root display server')
|
|
return old_resolve(self, *args, **kwargs)
|
|
monkeypatch.setattr(Path, 'resolve', resolve)
|
|
monkeypatch.setattr(a, 'Path', lambda value: tmp_path if value == '/proc' else Path(value))
|
|
monkeypatch.setattr(a.b, 'digest', lambda _: 'pinned-file-hash')
|
|
if allowed:
|
|
identity = a.desktop_identity(7)
|
|
assert identity['executable_identity_scope'] == 'root-Xorg-public-metadata; actual-executable-unverified'
|
|
assert identity['starttime'] == 123 and identity['uid'] == 0
|
|
assert identity['command_sha256'] == a.hashlib.sha256(command).hexdigest()
|
|
else:
|
|
with pytest.raises(PermissionError):
|
|
a.desktop_identity(7)
|
|
|
|
|
|
def test_preimport_requires_all_binding_pins_without_evaluation_recorder(monkeypatch):
|
|
binding = SimpleNamespace(REQUIRED_RUNTIME={'runtime': None},
|
|
ISAAC_PINS={'runtime': None, 'lazy.physx': None})
|
|
imported = []
|
|
monkeypatch.setattr(a.importlib, 'import_module', lambda name: imported.append(name))
|
|
a.preimport_binding_runtime(binding)
|
|
assert imported == ['runtime', 'lazy.physx']
|
|
def fail(name):
|
|
if name == 'lazy.physx':
|
|
raise ImportError('missing exact backend')
|
|
monkeypatch.setattr(a.importlib, 'import_module', fail)
|
|
with pytest.raises(ImportError, match='exact backend'):
|
|
a.preimport_binding_runtime(binding)
|
|
|
|
|
|
def write(path, value):
|
|
path.write_bytes(a.canonical(value))
|
|
|
|
|
|
def test_cli_prepare_and_run_require_new_exact_grants_before_import_or_output(tmp_path, monkeypatch):
|
|
import train_cycle_sim_rootxy_v2 as cli
|
|
monkeypatch.setattr(cli, 'cpu_protocol', lambda: pytest.fail('No CPU/Kit import on denied admission'))
|
|
monkeypatch.setattr(a, 'supervise', lambda _: pytest.fail('No supervisor without grant'))
|
|
for mode in ('--run', '--prepare'):
|
|
with pytest.raises(PermissionError, match='admission INCOMPLETE'):
|
|
cli.main([mode, '--output', str(tmp_path / 'new')])
|
|
assert not (tmp_path / 'new').exists()
|
|
|
|
|
|
def test_physics_attempted_completed_reset_partial_failure_and_caps():
|
|
events = []
|
|
class Sim:
|
|
fail = False
|
|
def step(self):
|
|
events.append('actual')
|
|
if self.fail:
|
|
raise RuntimeError('simstepfailure')
|
|
class Env:
|
|
def reset(self):
|
|
return self._reset_idx()
|
|
def _reset_idx(self):
|
|
return Sim().step()
|
|
original_step, original_reset = Sim.step, Env.reset
|
|
hooks = runtime.RootXYPhysicsHooks(Sim, Env, lambda: None, lambda x: events.append(dict(x)))
|
|
with hooks:
|
|
Env().reset()
|
|
assert hooks.initialization == hooks.completed == 1
|
|
hooks.mode = 'rollout'
|
|
sim = Sim()
|
|
sim.fail = True
|
|
with pytest.raises(RuntimeError, match='simstepfailure'):
|
|
sim.step()
|
|
assert hooks.attempted == 2 and hooks.completed == 1 and hooks.rollout == 1
|
|
hooks.rollout = 72460
|
|
with pytest.raises(RuntimeError, match='cap'):
|
|
Sim().step()
|
|
assert Sim.step is original_step and Env.reset is original_reset
|
|
assert events[0]['attempted_shared_calls'] == 1 and events[1] == 'actual'
|
|
|
|
|
|
def test_physics_safety_missing_reset_sample_is_fatal_and_prep_zero_calls():
|
|
class Sim:
|
|
def step(self):
|
|
pass
|
|
class Env:
|
|
def reset(self):
|
|
Sim().step()
|
|
_reset_idx = reset
|
|
audit = SimpleNamespace(samples=0, check=lambda: None, report=lambda: {})
|
|
hooks = runtime.RootXYPhysicsHooks(Sim, Env, lambda: None, lambda _: None)
|
|
with hooks:
|
|
hooks.attach_audit(audit)
|
|
with pytest.raises(RuntimeError, match='safety sample'):
|
|
Env().reset()
|
|
assert hooks.attempted == hooks.completed == 1
|
|
with runtime.RootXYPhysicsHooks(Sim, Env, lambda: None, lambda _: None, preparation=True) as hooks:
|
|
with pytest.raises(PermissionError, match='PREP'):
|
|
Sim().step()
|
|
assert hooks.attempted == hooks.completed == 0
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['start', 'partial_start', 'construct', 'identity', 'learn',
|
|
'close_env', 'postclose_identity', 'publication', 'close_app', None])
|
|
def test_worker_lifecycle_failures_are_provisional_and_owner_closes(fault):
|
|
events, published = [], []
|
|
class API:
|
|
started = False
|
|
physics_hooks = None
|
|
def start(self):
|
|
events.append('start')
|
|
if fault == 'start':
|
|
raise RuntimeError(fault)
|
|
self.started = True
|
|
if fault == 'partial_start':
|
|
raise RuntimeError(fault)
|
|
@contextlib.contextmanager
|
|
def physics(self):
|
|
events.append('hooks_enter')
|
|
yield SimpleNamespace(attempted=0, attach_audit=lambda _: None, safety_boundary=lambda: None,
|
|
report=lambda: {})
|
|
events.append('hooks_exit')
|
|
def construct(self):
|
|
events.append('construct')
|
|
if fault == 'construct':
|
|
raise RuntimeError(fault)
|
|
def check_actual(self):
|
|
events.append('actual')
|
|
@contextlib.contextmanager
|
|
def safety(self):
|
|
yield object()
|
|
def learn(self):
|
|
events.append('learn')
|
|
if fault == 'learn':
|
|
raise RuntimeError(fault)
|
|
return {'status': 'completed_provisional', 'training_complete': True,
|
|
'candidate': {'path': 'forensic.pt'}, 'accepted_skill': False}
|
|
def close_env(self):
|
|
events.append('close_env')
|
|
if fault == 'close_env':
|
|
raise RuntimeError(fault)
|
|
def close_app(self, code):
|
|
events.append('close_app')
|
|
if fault == 'close_app':
|
|
raise RuntimeError(fault)
|
|
api = API()
|
|
def verify():
|
|
events.append('verify')
|
|
if fault == 'identity' or fault == 'postclose_identity' and 'close_env' in events:
|
|
raise RuntimeError(fault)
|
|
def publish(value):
|
|
published.append(dict(value))
|
|
if fault == 'publication':
|
|
raise OSError(fault)
|
|
call = lambda: runtime.worker_lifecycle(api, prepare=False, guard=lambda: None,
|
|
verify=verify, publish=publish)
|
|
if fault is None:
|
|
assert call()['status'] == 'completed_provisional'
|
|
assert events.index('construct') < events.index('actual') < events.index('learn')
|
|
else:
|
|
with pytest.raises((RuntimeError, OSError)):
|
|
call()
|
|
assert published[-1]['candidate'] is None and not published[-1]['training_complete']
|
|
assert 'close_env' in events and 'close_app' in events
|
|
assert events.index('close_env') < events.index('close_app')
|
|
if 'hooks_exit' in events:
|
|
assert events.index('close_env') < events.index('close_app') < events.index('hooks_exit')
|
|
|
|
|
|
def test_guard_deadline_cancellation_storage_and_atomic_write_failure(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(a.b, 'boottime', lambda: 10.)
|
|
budget = a.GuardBudget(tmp_path, 10., 'run')
|
|
budget.write(tmp_path / 'value.json', {'ok': True})
|
|
monkeypatch.setattr(a.b, 'boottime', lambda: 7090.)
|
|
with pytest.raises(RuntimeError, match='deadline'):
|
|
budget.check()
|
|
budget.check(cleanup=True)
|
|
monkeypatch.setattr(a.b, 'boottime', lambda: 10.)
|
|
budget.interrupted = True
|
|
with pytest.raises(RuntimeError, match='cancelled'):
|
|
budget.check()
|
|
budget.interrupted = False
|
|
budget.external_growth = a.CAP
|
|
with pytest.raises(RuntimeError, match='storage'):
|
|
budget.check()
|
|
budget.external_growth = 0
|
|
monkeypatch.setattr(a.os, 'replace', lambda *_: (_ for _ in ()).throw(OSError('atomic failure')))
|
|
with pytest.raises(OSError):
|
|
budget.write(tmp_path / 'value.json', {'ok': False})
|
|
assert a.read(tmp_path / 'value.json') == {'ok': True}
|
|
assert (tmp_path / 'value.json.pending').exists() # forensic bytes remain charged
|
|
|
|
|
|
def test_native_monitor_failure_is_not_swallowed(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(a.b, 'boottime', lambda: 10.)
|
|
monitor = SimpleNamespace(check=lambda: (_ for _ in ()).throw(RuntimeError('nativefail')))
|
|
with pytest.raises(RuntimeError, match='nativefail'):
|
|
a.GuardBudget(tmp_path, 10., 'run', monitor).check()
|
|
|
|
|
|
def test_source_hash_drift_is_rejected(tmp_path, monkeypatch):
|
|
(tmp_path / 'file').write_text('original')
|
|
manifest = {'format': a.PREREG_FORMAT, 'task_id': a.TASK_ID, 'launch_integration_complete': True,
|
|
'accepted_skill': False, 'sources': {'file': a.b.digest(tmp_path / 'file')},
|
|
'training': {'seeds': [92511], 'validation_seed': 93511, 'num_envs': 128,
|
|
'max_total_wall_s': 7200, 'cleanup_reserve_s': 120, 'max_artifact_bytes': a.CAP,
|
|
'max_validation_transitions': 806400, 'validation_updates': [64, 128, 192, 256]}}
|
|
monkeypatch.setattr(a, 'ROOT', tmp_path)
|
|
a.verify_sources(manifest)
|
|
(tmp_path / 'file').write_text('drift')
|
|
with pytest.raises(ValueError, match='source drift'):
|
|
a.verify_sources(manifest)
|
|
|
|
|
|
@pytest.fixture
|
|
def supervisor(tmp_path, monkeypatch):
|
|
root = tmp_path / 'run'
|
|
args = SimpleNamespace(prepare=True, output=root, manifest=tmp_path / 'manifest.json',
|
|
trusted_manifest_sha256='m')
|
|
write(args.manifest, {'sources': {}})
|
|
manifest, grant, policy = {'sources': {}}, {'gpu_identity': {'uuid': 'GPU-mock'}}, {}
|
|
monkeypatch.setattr(a, 'admission', lambda *_: (manifest, grant, policy))
|
|
monkeypatch.setattr(a, 'verify_sources', lambda *_: None)
|
|
monkeypatch.setattr(a.b, 'boottime', lambda: 10.)
|
|
monkeypatch.setattr(a.owner, 'launch_command', lambda *_: ['NOT_EXECUTED'])
|
|
monkeypatch.setattr(a, 'launch_identity', lambda: {})
|
|
monkeypatch.setattr(a.owner, 'bound_filesystem_inputs', lambda *_: {})
|
|
monkeypatch.setattr(a.owner, 'verify_filesystem_inputs', lambda *_: {})
|
|
monkeypatch.setattr(a.ctypes, 'CDLL', lambda *_ , **__: SimpleNamespace(prctl=lambda *_: 0))
|
|
monkeypatch.setattr(a.signal, 'signal', lambda *_: None)
|
|
monkeypatch.setattr(a.signal, 'pidfd_send_signal', lambda *_: None, raising=False)
|
|
monkeypatch.setattr(a.os, 'pidfd_open', lambda *_: None, raising=False)
|
|
monkeypatch.setattr(a, 'GPUOwnership', lambda _: SimpleNamespace(check=lambda *_ , **__: {}))
|
|
owned = SimpleNamespace(unknown=False, known=set(), attach=lambda _: None, discover=lambda: {})
|
|
monkeypatch.setattr(a.owner, 'OwnedProcesses', lambda: owned)
|
|
monitor = SimpleNamespace(check=lambda **_: None, latest={}, baseline={}, report=lambda: {})
|
|
monkeypatch.setattr(a.owner, 'NativeMonitor', lambda *_: monitor)
|
|
child = SimpleNamespace(pid=12345, stdout=SimpleNamespace(close=lambda: None), returncode=0,
|
|
poll=lambda: 0, wait=lambda **_: 0)
|
|
monkeypatch.setattr(a.subprocess, 'Popen', lambda *_ , **__: child)
|
|
capture = SimpleNamespace(eof=True, discarded=0, error=None, check=lambda: None,
|
|
pump=lambda: None, close=lambda: None)
|
|
monkeypatch.setattr(a.owner, 'StdoutCapture', lambda *_: capture)
|
|
monkeypatch.setattr(a.owner, 'cleanup_group', lambda *_: True)
|
|
def exchange(*_):
|
|
write(root / 'prepare-worker-result.json', {'status': 'prepared_config_only', 'physics_calls': 0,
|
|
'postclose_identity_verified': True})
|
|
write(root / 'training-binding.json', {})
|
|
write(root / 'config-preparation-provisional.json', {'status': 'prepared_config_only'})
|
|
monkeypatch.setattr(a.owner, 'exchange_worker', exchange)
|
|
return args, child, monitor, capture, owned
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['spawn', 'attach', 'handshake', 'workerexit', 'nonzero',
|
|
'nativefail', 'cleanupfailure', 'postexit', None])
|
|
def test_supervisor_independent_exit_native_and_cleanup_gate(supervisor, monkeypatch, fault):
|
|
args, child, monitor, capture, owned = supervisor
|
|
def fail(*_, **__):
|
|
raise RuntimeError(fault)
|
|
if fault == 'spawn':
|
|
monkeypatch.setattr(a.subprocess, 'Popen', fail)
|
|
elif fault == 'attach':
|
|
owned.attach = fail
|
|
elif fault == 'handshake':
|
|
monkeypatch.setattr(a.owner, 'exchange_worker', fail)
|
|
elif fault == 'workerexit':
|
|
child.wait = fail
|
|
elif fault == 'nonzero':
|
|
child.returncode = 1
|
|
elif fault == 'nativefail':
|
|
monitor.check = fail
|
|
elif fault == 'cleanupfailure':
|
|
monkeypatch.setattr(a.owner, 'cleanup_group', lambda *_: False)
|
|
elif fault == 'postexit':
|
|
monkeypatch.setattr(a.owner, 'verify_filesystem_inputs', fail)
|
|
if fault == 'nativefail':
|
|
with pytest.raises(RuntimeError, match='nativefail'):
|
|
a.supervise(args) # cannot publish through a failed monitor
|
|
else:
|
|
result = a.supervise(args)
|
|
assert result == (0 if fault is None else 1)
|
|
assert not (args.output / 'locked_candidate.json').exists()
|
|
if (args.output / 'prepare-supervisor.json').exists():
|
|
summary = a.read(args.output / 'prepare-supervisor.json')
|
|
assert summary['success'] is (fault is None)
|
|
|
|
|
|
def eligible_rows(xy=.02):
|
|
return [{'env': i, 'success': True, 'physics_samples': 4500, 'complete_clock': True,
|
|
'rootxy_rms_m': xy, 'rootxy_max_m': xy, 'rootxy_breached': False, 'safety_ok': True,
|
|
'height_rms_m': .01, 'swing_rms_m': .01, 'landing_error_m': .01,
|
|
'functional_audit': {'sim_functional_passed': True}} for i in range(128)]
|
|
|
|
|
|
def test_candidate_commit_independently_rechecks_eligibility_and_publication(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(a.b, 'boottime', lambda: 10.)
|
|
budget = a.GuardBudget(tmp_path, 10., 'run')
|
|
write(tmp_path / 'manifest.json', {})
|
|
for update in (64, 128, 192, 256):
|
|
write(tmp_path / f'validation_u{update:04d}.json', {'rows': eligible_rows()})
|
|
path = tmp_path / 'checkpoint_u0064.pt'
|
|
path.write_bytes(b'forensic checkpoint bytes')
|
|
key = [.02, .01 / .06 + .01 / .025 + .01 / .05, 64]
|
|
# Match the exact shared floating reduction, not a rounded displayed metric.
|
|
key[0] = sum(.02 for _ in range(128)) / 128
|
|
key[1] = sum(key[1] for _ in range(128)) / 128
|
|
result = {'status': 'completed_provisional', 'training_complete': True, 'accepted_skill': False,
|
|
'checkpoints_selectable': False, 'postclose_identity_verified': True,
|
|
'candidate': {'path': path.name, 'sha256': a.b.digest(path), 'update': 64, 'selection_key': key}}
|
|
lock = a.commit_candidate(tmp_path, result, {}, budget)
|
|
assert lock['accepted_skill'] is False
|
|
(tmp_path / 'locked_candidate.json').unlink()
|
|
rows = eligible_rows()
|
|
rows[4]['functional_audit']['sim_functional_passed'] = False
|
|
write(tmp_path / 'validation_u0128.json', {'rows': rows})
|
|
with pytest.raises(ValueError, match='functional'):
|
|
a.commit_candidate(tmp_path, result, {}, budget)
|
|
assert not (tmp_path / 'locked_candidate.json').exists()
|
|
write(tmp_path / 'validation_u0128.json', {'rows': eligible_rows()})
|
|
monkeypatch.setattr(budget, 'write', lambda *_ , **__: (_ for _ in ()).throw(OSError('final commit')))
|
|
with pytest.raises(OSError, match='final commit'):
|
|
a.commit_candidate(tmp_path, result, {}, budget)
|
|
assert path.exists() and not (tmp_path / 'locked_candidate.json').exists()
|
|
|
|
|
|
@pytest.mark.parametrize('mutation', [None, 'old_format', 'wrong_task', 'expired', 'long_admission',
|
|
'fake_p2_in_prep', 'extra_field', 'bad_hash', 'run_missing_digest'])
|
|
def test_prepare_exact_grant_schema_and_expiry(tmp_path, monkeypatch, mutation):
|
|
root = tmp_path / 'new'
|
|
manifest = tmp_path / 'manifest.json'
|
|
grant_path = tmp_path / 'grant.json'
|
|
write(manifest, {})
|
|
monkeypatch.setattr(a, 'verify_sources', lambda _: None)
|
|
monkeypatch.setattr(a, 'launch_identity', lambda: {})
|
|
monkeypatch.setattr(a, 'policy_check', lambda *_: {'not_after_boottime': 9000.})
|
|
grant = {'format': a.GRANT_FORMAT, 'profile': a.PROFILE, 'task_id': a.TASK_ID, 'mode': 'prepare',
|
|
'artifact_root': str(root), 'manifest_sha256': a.b.digest(manifest),
|
|
'storage_policy_sha256': 'p', 'launch_identity_sha256': a.sha({}),
|
|
'boot_id': a.b.boot_id(), 'not_after_boottime': 1000., 'parent_review': 'config-only',
|
|
'preparation_sha256': None, 'binding_sha256': None, 'p2_decision_sha256': None,
|
|
'gpu_identity': {}}
|
|
if mutation == 'old_format':
|
|
grant['format'] = 'skillfactory.evaluation-parent-grant.v2'
|
|
elif mutation == 'wrong_task':
|
|
grant['task_id'] = 'v1'
|
|
elif mutation == 'expired':
|
|
grant['not_after_boottime'] = 10.
|
|
elif mutation == 'long_admission':
|
|
grant['not_after_boottime'] = 3611.
|
|
elif mutation == 'fake_p2_in_prep':
|
|
grant['p2_decision_sha256'] = '0' * 64
|
|
elif mutation == 'extra_field':
|
|
grant['approved'] = True
|
|
elif mutation == 'run_missing_digest':
|
|
grant['mode'] = 'run'
|
|
write(grant_path, grant)
|
|
args = SimpleNamespace(prepare=mutation != 'run_missing_digest', output=root, manifest=manifest,
|
|
trusted_manifest_sha256=a.b.digest(manifest), authorization=grant_path,
|
|
trusted_authorization_sha256='0' * 64 if mutation == 'bad_hash' else a.b.digest(grant_path),
|
|
storage_policy=tmp_path / 'policy', trusted_storage_policy_sha256='p')
|
|
if mutation is None:
|
|
_, actual, _ = a.admission(args, 10.)
|
|
assert actual == grant and not root.exists()
|
|
else:
|
|
with pytest.raises((ValueError, PermissionError)):
|
|
a.admission(args, 10.)
|
|
assert not root.exists()
|
|
|
|
|
|
def test_native_policy_exact_horizon_is_distinct_from_admission_expiry(tmp_path, monkeypatch):
|
|
root = tmp_path / 'new'
|
|
policy_path = tmp_path / 'policy.json'
|
|
monkeypatch.setattr(a.owner, 'mount_identity', lambda _: 'mount')
|
|
policy = {'format': a.POLICY_FORMAT, 'profile': a.PROFILE, 'artifact_root': str(root), 'limit_bytes': a.CAP,
|
|
'native_paths': a.owner.native_paths(root), 'shared_native_paths': a.owner.shared_native_paths(),
|
|
'task_owned_cache_paths': sorted([*a.redirects(root, 'prepare').values(), *a.redirects(root, 'run').values()]),
|
|
'free_space_paths': a.owner.free_space_paths(root),
|
|
'free_space_mount_identities': {p: 'mount' for p in a.owner.free_space_paths(root)},
|
|
'mount_identity': 'mount', 'check_interval_seconds': 1., 'max_native_growth_bytes': a.CAP,
|
|
'min_free_bytes': 2 * a.CAP, 'coverage_gaps': list(a.owner.NATIVE_GAPS), 'native_hard_cap': False,
|
|
'boot_id': a.b.boot_id(), 'issued_boottime': 10., 'not_after_boottime': 10810., 'lifetime_seconds': 10800}
|
|
args = SimpleNamespace(output=root, storage_policy=policy_path)
|
|
for duration in (10800, 3600, 10801):
|
|
policy.update(lifetime_seconds=duration, not_after_boottime=10. + duration)
|
|
write(policy_path, policy)
|
|
args.trusted_storage_policy_sha256 = a.b.digest(policy_path)
|
|
if duration == 10800:
|
|
assert a.policy_check(args, 10.) == policy
|
|
else:
|
|
with pytest.raises(PermissionError):
|
|
a.policy_check(args, 10.)
|
|
|
|
|
|
def test_fast_app_shutdown_cannot_mask_ledger_teardown_failure():
|
|
api = a.RuntimeAPI.__new__(a.RuntimeAPI)
|
|
codes = []
|
|
api.ledger = SimpleNamespace(__exit__=lambda: (_ for _ in ()).throw(OSError('ledger close')))
|
|
api.launcher = SimpleNamespace(_app=SimpleNamespace(close=lambda **kw: codes.append(kw['exit_code'])))
|
|
with pytest.raises(OSError, match='ledger close'):
|
|
api.close_app(0)
|
|
assert codes == [1]
|
|
|
|
|
|
def test_supervisor_final_candidate_commit_failure_removes_lock(supervisor, monkeypatch):
|
|
args, child, monitor, capture, owned = supervisor
|
|
args.prepare = False
|
|
args.output.mkdir()
|
|
write(args.output / 'manifest.json', {})
|
|
write(args.output / 'training-binding.json', {})
|
|
(args.output / 'checkpoint_u0064.pt').write_bytes(b'forensic')
|
|
monkeypatch.setattr(a, 'validate_completion', lambda *_: None)
|
|
def exchange(*_):
|
|
write(args.output / 'run-worker-result.json', {'status': 'completed_provisional',
|
|
'training_complete': True, 'postclose_identity_verified': True})
|
|
monkeypatch.setattr(a.owner, 'exchange_worker', exchange)
|
|
def fail_commit(root, *_):
|
|
(root / 'locked_candidate.json').write_bytes(b'failed durable commit')
|
|
raise OSError('candidate commit failure')
|
|
monkeypatch.setattr(a, 'commit_candidate', fail_commit)
|
|
with pytest.raises(OSError, match='candidate commit failure'):
|
|
a.supervise(args)
|
|
assert not (args.output / 'locked_candidate.json').exists()
|
|
assert (args.output / 'checkpoint_u0064.pt').read_bytes() == b'forensic'
|
|
assert a.read(args.output / 'run-supervisor.json')['success'] is False
|
|
|
|
|
|
def test_worker_fd_rejects_non_supervisor_peer_before_handoff():
|
|
parent, child = a.socket.socketpair()
|
|
try:
|
|
with pytest.raises(PermissionError, match='Authenticated RootXY supervisor'):
|
|
a.run_worker(child.detach())
|
|
finally:
|
|
parent.close()
|
|
child.close()
|
|
|
|
|
|
@pytest.mark.parametrize('fault', [None, 'pending', 'safety_coverage', 'physics_partial', 'startup_cap', 'baseline'])
|
|
def test_independent_completion_debits_and_all_tick_safety(tmp_path, monkeypatch, fault):
|
|
data = {'reserved_transitions': 1048576, 'reservations': 256, 'pending': None, 'updates': {'92511': 256},
|
|
'validation_reserved_transitions': 806400, 'validation_attempted_control_calls': 5400,
|
|
'validation_completed_control_calls': 5400,
|
|
'validation_evaluations_started': ['analytic_zero', 'frozen_u256', 'u0064', 'u0128', 'u0192', 'u0256'],
|
|
'clock': {'start_s': 10., 'deadline_s': 7210., 'boot_id': a.b.boot_id()}}
|
|
safety = {'physics_samples': 67960, 'finite': True, 'max_effort_velocity_ratio': 0.,
|
|
'max_joint_excess_rad': 0., 'max_guide_vertical_N': 0.}
|
|
physics = {'attempted_shared_calls': 67962, 'completed_shared_calls': 67962,
|
|
'initialization_attempted_shared_calls': 2, 'rollout_attempted_shared_calls': 67960,
|
|
'startup_completed_without_task_audit': 2, 'safety': safety}
|
|
rows = eligible_rows()
|
|
if fault == 'pending':
|
|
data['pending'] = {'update': 256, 'seed': 92511}
|
|
elif fault == 'safety_coverage':
|
|
safety['physics_samples'] -= 1
|
|
elif fault == 'physics_partial':
|
|
physics['completed_shared_calls'] -= 1
|
|
elif fault == 'startup_cap':
|
|
physics['initialization_attempted_shared_calls'] = 1001
|
|
elif fault == 'baseline':
|
|
rows[0]['success'] = False
|
|
write(tmp_path / 'budget.json', data)
|
|
for name in ('analytic_zero', 'frozen_u256'):
|
|
write(tmp_path / ('validation_' + name + '.json'), {'rows': rows})
|
|
result = {'budget': data, 'physics': physics}
|
|
manifest = {'training': {'safety_config': 'configs/contact_cycle/acceptance_v1.json'}}
|
|
if fault is None:
|
|
a.validate_completion(tmp_path, result, manifest, 10.)
|
|
else:
|
|
with pytest.raises(ValueError):
|
|
a.validate_completion(tmp_path, result, manifest, 10.)
|