142 lines
6.5 KiB
Python
142 lines
6.5 KiB
Python
"""Synthetic counterexamples for evidence checking, not physical success proof."""
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import torch
|
|
from SkillFactory.legs.four_skills.forces import ForceSchedule
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
spec = importlib.util.spec_from_file_location('four_auditor', ROOT / 'scripts/audit_four_skills.py')
|
|
auditor = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(auditor)
|
|
|
|
|
|
def js(path, value):
|
|
path.write_text(json.dumps(value))
|
|
|
|
|
|
def smooth(x):
|
|
x = x.clamp(0, 1)
|
|
return x.square() * (3 - 2*x)
|
|
|
|
|
|
@pytest.fixture
|
|
def run(tmp_path):
|
|
out = tmp_path / 'analytic'
|
|
out.mkdir()
|
|
sizes = [350, 1000, 950, 1850, 350]
|
|
kinds = [0, 1, 2, 3, 0]
|
|
count = sum(sizes)
|
|
skill = torch.cat([torch.full((n,), k, dtype=torch.long) for k, n in zip(kinds, sizes)])
|
|
age = torch.cat([torch.arange(1, n+1).float()*.002 for n in sizes])
|
|
foot = torch.zeros(count, 1, 3)
|
|
foot[..., 2] = .02
|
|
fn = torch.ones(count, 1) * 50
|
|
a, b, c, d = 350, 1350, 2300, 4150
|
|
foot[a:b, 0, 2] += .05 * smooth((age[a:b]-1)/.6)
|
|
fn[a:b, 0] *= 1-smooth(age[a:b])
|
|
p = smooth(age[b:c]/1.4)
|
|
foot[b:c, 0, :2] = p[:, None]*torch.tensor([.024, .010])
|
|
foot[b:c, 0, 2] = .07
|
|
fn[b:c] = 0
|
|
foot[c:, 0, :2] = torch.tensor([.024, .010])
|
|
foot[c:d, 0, 2] += .05*(1-smooth(age[c:d]/.8))
|
|
fn[c:d, 0] *= smooth((age[c:d]-.8)/2.2)
|
|
normal = torch.zeros(count, 1, 3)
|
|
normal[..., 2] = fn
|
|
zero = torch.zeros(count, 1)
|
|
t = {'tick': torch.arange(1, count+1)[:, None],
|
|
'time_s': (torch.arange(1, count+1).float()*.002)[:, None],
|
|
'skill': skill[:, None], 'age_s': age[:, None], 'episode_id': torch.ones(count, 1, dtype=torch.long),
|
|
'weight': torch.ones(count, 1)*50, 'failed': zero.bool(), 'root': torch.zeros(count, 1, 13),
|
|
'joint_pos': torch.zeros(count, 1, 3), 'joint_vel': torch.zeros(count, 1, 3),
|
|
'normal': normal, 'foot': foot, 'gap': foot[..., 2]-.02, 'target': foot.clone(),
|
|
'reference': torch.zeros(count, 1, 3), 'q_ik': torch.zeros(count, 1, 3),
|
|
'actor_observation': torch.zeros(count, 1, 48), 'external': torch.zeros(count, 1, 2, 6),
|
|
'guide': torch.zeros(count, 1, 2, 6), 'auxiliary': torch.zeros(count, 1, 2, 6),
|
|
'phase': torch.where(skill == 0, 0, torch.where(skill == 1, torch.where(age < 1., 1, 2),
|
|
torch.where(skill == 2, 3, torch.where(age < .8, 4, 5))))[:, None],
|
|
'limit_ratio': zero.clone(),
|
|
'joint_excess': zero.clone(), 'height_error': zero.clone(), 'tilt': zero.clone()}
|
|
initial = {k: v[0].clone() for k, v in t.items()}
|
|
initial['time_s'].zero_()
|
|
torch.save(t, out/'trace.pt')
|
|
torch.save(initial, out/'initial-state.pt')
|
|
approval = {'seed': 115011, 'sequence': 0, 'loops': 1, 'mode': 'evaluate', 'tier': 'none',
|
|
'config_sha256': hashlib.sha256((ROOT/'configs/four_skills/v050.json').read_bytes()).hexdigest(),
|
|
'controller': 'analytic', 'checkpoint': None, 'checkpoint_sha256': None}
|
|
js(out/'approval.json', approval)
|
|
counters = {'resets': 11, 'pose_writes': 14}
|
|
js(out/'worker-result.json', {'actual_ticks': count, 'requested_loops': 1, 'completed_loops': 1,
|
|
'functional_success': True, 'initialization_counters': counters, 'final_counters': counters,
|
|
'physics_counts': {'initialization': 0, 'completed': count}})
|
|
js(out/'supervisor-result.json', {'success': True, 'owned_cleanup': True, 'waited_exit': True})
|
|
js(out/'runtime-identity.json', {'payload_body_index': 0, 'body_names': ['virtual_payload', 'foot']})
|
|
js(out/'force-schedule.json', ForceSchedule(115011, [0], 'none', 'cpu', episode_ids=[1]).manifest())
|
|
js(out/'transitions.json', [dict(zip(('from', 'to', 'at_tick'), (old, new, tick)),
|
|
feedback_ready=True, feedback_hold_complete=True) for old, new, tick in
|
|
[('support', 'retract', a), ('retract', 'swing', b), ('swing', 'plant', c), ('plant', 'support', d)]])
|
|
assert auditor.audit(out)['passed']
|
|
return out
|
|
|
|
|
|
def change_trace(run, change):
|
|
t = torch.load(run/'trace.pt', weights_only=True)
|
|
change(t)
|
|
torch.save(t, run/'trace.pt')
|
|
return auditor.audit(run)
|
|
|
|
|
|
def test_actual_tensor_skill_order_cannot_be_replaced_by_claimed_json(run):
|
|
result = change_trace(run, lambda t: t['skill'].__setitem__(slice(350, 1350), 3))
|
|
assert not result['passed'] and 'actual_skill_order_mismatch' in result['failure_reasons']
|
|
|
|
|
|
@pytest.mark.parametrize('field', ['time_s', 'episode_id'])
|
|
def test_timestamps_and_episode_identity_are_not_unchecked_labels(run, field):
|
|
result = change_trace(run, lambda t: t[field].__setitem__((100, 0), t[field][100, 0]+1))
|
|
assert not result['passed']
|
|
|
|
|
|
def test_transition_tick_must_equal_actual_tensor_boundary(run):
|
|
edges = json.loads((run/'transitions.json').read_text())
|
|
edges[0]['at_tick'] += 1
|
|
js(run/'transitions.json', edges)
|
|
assert 'transition_tick_mismatch' in auditor.audit(run)['failure_reasons']
|
|
|
|
|
|
def test_support_ratio_alone_cannot_hide_interrupted_continuous_hold(run):
|
|
result = change_trace(run, lambda t: t['normal'].__setitem__((340, 0, 2), 0.))
|
|
assert not result['passed']
|
|
assert any('continuous_support' in reason for reason in result['failure_reasons'])
|
|
|
|
|
|
def test_swing_progress_must_persist_through_hold_not_only_last_frame(run):
|
|
result = change_trace(run, lambda t: t['foot'].__setitem__((slice(2100, 2299), 0, slice(0, 2)), 0.))
|
|
assert any('continuous_signed_swing_hold' in reason for reason in result['failure_reasons'])
|
|
|
|
|
|
def test_zero_requested_couple_is_verified_not_just_net_force(run):
|
|
result = change_trace(run, lambda t: t['external'].__setitem__((20, 0, 0, 3), .1))
|
|
assert 'sampled_force_not_actually_applied' in result['failure_reasons']
|
|
|
|
|
|
def test_schedule_tier_must_equal_approval(run):
|
|
js(run/'force-schedule.json', ForceSchedule(115011, [0], 'small', 'cpu', episode_ids=[1]).manifest())
|
|
assert 'schedule_tier_not_approved' in auditor.audit(run)['failure_reasons']
|
|
|
|
|
|
def test_same_initial_tensors_and_force_schedule_do_not_pair_different_tasks(run, tmp_path):
|
|
import shutil
|
|
other = tmp_path/'trained'
|
|
shutil.copytree(run, other)
|
|
a = json.loads((other/'approval.json').read_text())
|
|
a.update(controller='trained', checkpoint='/example/weights.pt', checkpoint_sha256='a'*64, sequence=1)
|
|
js(other/'approval.json', a)
|
|
result = auditor.compare_pair(run, other)
|
|
assert not result['paired'] and not result['approved_conditions_equal']['sequence']
|