Files
SkillFactory/tests/test_four_skills_audit.py

142 lines
6.5 KiB
Python

"""Synthetic counterexamples for evidence checking, not physical success proof."""
import copy
import hashlib
import importlib.util
import json
from pathlib import Path
import pytest
import torch
from SkillFactory.legs.four_skills.forces import ForceSchedule
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location('four_auditor', ROOT / 'scripts/audit_four_skills.py')
auditor = importlib.util.module_from_spec(spec)
spec.loader.exec_module(auditor)
def js(path, value):
path.write_text(json.dumps(value))
def smooth(x):
x = x.clamp(0, 1)
return x.square() * (3 - 2*x)
@pytest.fixture
def run(tmp_path):
out = tmp_path / 'analytic'
out.mkdir()
sizes = [350, 1000, 950, 1850, 350]
kinds = [0, 1, 2, 3, 0]
count = sum(sizes)
skill = torch.cat([torch.full((n,), k, dtype=torch.long) for k, n in zip(kinds, sizes)])
age = torch.cat([torch.arange(1, n+1).float()*.002 for n in sizes])
foot = torch.zeros(count, 1, 3)
foot[..., 2] = .02
fn = torch.ones(count, 1) * 50
a, b, c, d = 350, 1350, 2300, 4150
foot[a:b, 0, 2] += .05 * smooth((age[a:b]-1)/.6)
fn[a:b, 0] *= 1-smooth(age[a:b])
p = smooth(age[b:c]/1.4)
foot[b:c, 0, :2] = p[:, None]*torch.tensor([.024, .010])
foot[b:c, 0, 2] = .07
fn[b:c] = 0
foot[c:, 0, :2] = torch.tensor([.024, .010])
foot[c:d, 0, 2] += .05*(1-smooth(age[c:d]/.8))
fn[c:d, 0] *= smooth((age[c:d]-.8)/2.2)
normal = torch.zeros(count, 1, 3)
normal[..., 2] = fn
zero = torch.zeros(count, 1)
t = {'tick': torch.arange(1, count+1)[:, None],
'time_s': (torch.arange(1, count+1).float()*.002)[:, None],
'skill': skill[:, None], 'age_s': age[:, None], 'episode_id': torch.ones(count, 1, dtype=torch.long),
'weight': torch.ones(count, 1)*50, 'failed': zero.bool(), 'root': torch.zeros(count, 1, 13),
'joint_pos': torch.zeros(count, 1, 3), 'joint_vel': torch.zeros(count, 1, 3),
'normal': normal, 'foot': foot, 'gap': foot[..., 2]-.02, 'target': foot.clone(),
'reference': torch.zeros(count, 1, 3), 'q_ik': torch.zeros(count, 1, 3),
'actor_observation': torch.zeros(count, 1, 48), 'external': torch.zeros(count, 1, 2, 6),
'guide': torch.zeros(count, 1, 2, 6), 'auxiliary': torch.zeros(count, 1, 2, 6),
'phase': torch.where(skill == 0, 0, torch.where(skill == 1, torch.where(age < 1., 1, 2),
torch.where(skill == 2, 3, torch.where(age < .8, 4, 5))))[:, None],
'limit_ratio': zero.clone(),
'joint_excess': zero.clone(), 'height_error': zero.clone(), 'tilt': zero.clone()}
initial = {k: v[0].clone() for k, v in t.items()}
initial['time_s'].zero_()
torch.save(t, out/'trace.pt')
torch.save(initial, out/'initial-state.pt')
approval = {'seed': 115011, 'sequence': 0, 'loops': 1, 'mode': 'evaluate', 'tier': 'none',
'config_sha256': hashlib.sha256((ROOT/'configs/four_skills/v050.json').read_bytes()).hexdigest(),
'controller': 'analytic', 'checkpoint': None, 'checkpoint_sha256': None}
js(out/'approval.json', approval)
counters = {'resets': 11, 'pose_writes': 14}
js(out/'worker-result.json', {'actual_ticks': count, 'requested_loops': 1, 'completed_loops': 1,
'functional_success': True, 'initialization_counters': counters, 'final_counters': counters,
'physics_counts': {'initialization': 0, 'completed': count}})
js(out/'supervisor-result.json', {'success': True, 'owned_cleanup': True, 'waited_exit': True})
js(out/'runtime-identity.json', {'payload_body_index': 0, 'body_names': ['virtual_payload', 'foot']})
js(out/'force-schedule.json', ForceSchedule(115011, [0], 'none', 'cpu', episode_ids=[1]).manifest())
js(out/'transitions.json', [dict(zip(('from', 'to', 'at_tick'), (old, new, tick)),
feedback_ready=True, feedback_hold_complete=True) for old, new, tick in
[('support', 'retract', a), ('retract', 'swing', b), ('swing', 'plant', c), ('plant', 'support', d)]])
assert auditor.audit(out)['passed']
return out
def change_trace(run, change):
t = torch.load(run/'trace.pt', weights_only=True)
change(t)
torch.save(t, run/'trace.pt')
return auditor.audit(run)
def test_actual_tensor_skill_order_cannot_be_replaced_by_claimed_json(run):
result = change_trace(run, lambda t: t['skill'].__setitem__(slice(350, 1350), 3))
assert not result['passed'] and 'actual_skill_order_mismatch' in result['failure_reasons']
@pytest.mark.parametrize('field', ['time_s', 'episode_id'])
def test_timestamps_and_episode_identity_are_not_unchecked_labels(run, field):
result = change_trace(run, lambda t: t[field].__setitem__((100, 0), t[field][100, 0]+1))
assert not result['passed']
def test_transition_tick_must_equal_actual_tensor_boundary(run):
edges = json.loads((run/'transitions.json').read_text())
edges[0]['at_tick'] += 1
js(run/'transitions.json', edges)
assert 'transition_tick_mismatch' in auditor.audit(run)['failure_reasons']
def test_support_ratio_alone_cannot_hide_interrupted_continuous_hold(run):
result = change_trace(run, lambda t: t['normal'].__setitem__((340, 0, 2), 0.))
assert not result['passed']
assert any('continuous_support' in reason for reason in result['failure_reasons'])
def test_swing_progress_must_persist_through_hold_not_only_last_frame(run):
result = change_trace(run, lambda t: t['foot'].__setitem__((slice(2100, 2299), 0, slice(0, 2)), 0.))
assert any('continuous_signed_swing_hold' in reason for reason in result['failure_reasons'])
def test_zero_requested_couple_is_verified_not_just_net_force(run):
result = change_trace(run, lambda t: t['external'].__setitem__((20, 0, 0, 3), .1))
assert 'sampled_force_not_actually_applied' in result['failure_reasons']
def test_schedule_tier_must_equal_approval(run):
js(run/'force-schedule.json', ForceSchedule(115011, [0], 'small', 'cpu', episode_ids=[1]).manifest())
assert 'schedule_tier_not_approved' in auditor.audit(run)['failure_reasons']
def test_same_initial_tensors_and_force_schedule_do_not_pair_different_tasks(run, tmp_path):
import shutil
other = tmp_path/'trained'
shutil.copytree(run, other)
a = json.loads((other/'approval.json').read_text())
a.update(controller='trained', checkpoint='/example/weights.pt', checkpoint_sha256='a'*64, sequence=1)
js(other/'approval.json', a)
result = auditor.compare_pair(run, other)
assert not result['paired'] and not result['approved_conditions_equal']['sequence']