Files
SkillFactory/scripts/audit_four_skills.py
T

217 lines
14 KiB
Python

#!/usr/bin/env python3
"""Independent CPU-only audit of complete four-skill original500Hz traces.
This checks physical tensors, not the actor's reward or ready/success labels.
It never launches Kit, creates authority, alters traces or selects a candidate.
"""
import argparse
import importlib.util
import json
import math
from pathlib import Path
import sys
import torch
ROOT = Path(__file__).resolve().parents[1]
def audit(run):
run = Path(run)
config = json.loads((ROOT / 'configs/four_skills/v050.json').read_text())
approval = json.loads((run / 'approval.json').read_text())
worker = json.loads((run / 'worker-result.json').read_text())
supervisor = json.loads((run / 'supervisor-result.json').read_text())
trace = torch.load(run / 'trace.pt', map_location='cpu', weights_only=True)
initial = torch.load(run / 'initial-state.pt', map_location='cpu', weights_only=True)
reasons = []
def require(test, reason):
if not bool(test):
reasons.append(reason)
count = len(trace['tick'])
require(count > 0, 'empty_trace')
require(torch.equal(trace['tick'][:, 0], torch.arange(1, count + 1)), 'physical_clock_not_contiguous')
require(count == worker.get('actual_ticks'), 'actual_call_trace_count_mismatch')
require(all(bool(torch.isfinite(v).all()) for v in trace.values() if v.is_floating_point()), 'nonfinite_evidence')
require(not trace['failed'].any(), 'runtime_failure_in_trace')
require(trace['limit_ratio'].max() <= 1.00001, 'effort_velocity')
require(trace['joint_excess'].max() <= .01, 'joint_limits')
require(trace['height_error'].max() <= .1, 'root_height')
require(trace['tilt'].max() <= .6, 'tilt')
require(trace['gap'].min() >= -.002, 'penetration')
require(trace['guide'][..., 2].abs().max() == 0, 'vertical_guide')
if 'auxiliary' in trace:
auxiliary = trace['auxiliary']
az = auxiliary[..., 2]
require(az.sum(-1).max() <= 140., 'auxiliary_force')
require(az.min() >= -1e-6 and auxiliary[..., :2].abs().max() <= 1e-6
and auxiliary[..., 5].abs().max() <= 1e-6
and (auxiliary[..., 3:].norm(dim=-1) <= .06*az.clamp_min(0)+1e-6).all(), 'auxiliary_geometry')
else:
require(False, 'missing_auxiliary_evidence')
require(trace['normal'][..., 2].min() >= -.5, 'negative_normal')
require(worker.get('initialization_counters') == worker.get('final_counters'), 'reset_or_pose_write')
require(worker['physics_counts']['initialization'] == 0, 'unexpected_initialization_physics')
require(worker['physics_counts']['completed'] == count, 'unrecorded_physics')
transitions = json.loads((run / 'transitions.json').read_text())
sequence = config['sequences'][approval['sequence']]
loops = approval.get('loops') or (1 if approval['mode'] == 'smoke' else sequence['loops'])
require(worker.get('requested_loops') == loops, 'approved_loop_count_mismatch')
require(approval['config_sha256'] == __import__('hashlib').sha256((ROOT / 'configs/four_skills/v050.json').read_bytes()).hexdigest(), 'approved_config_changed')
expected_edges = [('support', 'retract'), ('retract', 'swing'), ('swing', 'plant'), ('plant', 'support')] * loops
require([(e['from'], e['to']) for e in transitions] == expected_edges, 'missing_or_illegal_transitions')
require(all(e['feedback_ready'] and e['feedback_hold_complete'] for e in transitions), 'switch_without_feedback')
kinds = trace['skill'][:, 0]
starts = [0] + (kinds[1:] != kinds[:-1]).nonzero().flatten().add(1).tolist()
ends = starts[1:] + [count]
require(len(starts) == 4 * loops + 1, 'incomplete_skill_segments')
require([int(kinds[i]) for i in starts] == [0, 1, 2, 3] * loops + [0], 'actual_skill_order_mismatch')
require(torch.equal(trace['time_s'][:, 0], trace['tick'][:, 0].float() * .002), 'time_tick_mismatch')
require(torch.equal(trace['episode_id'][:, 0], initial['episode_id'][0].expand(count)), 'episode_changed_during_composition')
for j, start in enumerate(starts[1:]):
if j < len(transitions):
require(transitions[j]['at_tick'] == int(trace['tick'][start, 0]) - 1, 'transition_tick_mismatch')
for start, end in zip(starts, ends):
expected_age = torch.arange(1, end - start + 1).float() * .002
require(torch.allclose(trace['age_s'][start:end, 0], expected_age, rtol=0., atol=.001), 'segment_age_mismatch')
age = trace['age_s'][:, 0]
expected_phase = torch.where(kinds == 0, 0, torch.where(kinds == 1,
torch.where(age < 1., 1, 2), torch.where(kinds == 2, 3, torch.where(age < .8, 4, 5))))
if 'phase' in trace:
require(torch.equal(trace['phase'][:, 0], expected_phase), 'intrinsic_phase_mismatch')
else:
require(False, 'missing_phase_evidence')
require(not (((expected_phase == 2) & (age > 1.54)) & (trace['gap'][:, 0] < .02)).any(), 'original_late_lift_floor')
require(not ((expected_phase == 3) & (trace['gap'][:, 0] < .01)).any(), 'original_swing_clearance')
require(not (((expected_phase == 5) & (age > 2.56)) &
(trace['normal'][:, 0, 2] < .5*trace['weight'][:, 0])).any(), 'original_reload_deadline')
holds = sequence['holds_s']
windows = []
swing_index = 0
for index, (start, end) in enumerate(zip(starts, ends)):
kind = int(kinds[start])
name = config['skills'][kind]
hold = holds[kind]
dwell = .2 if kind in (0, 3) else .1
window = math.ceil((hold + dwell) / .002) - 1
if end - start < window:
require(False, f'{index}:{name}:incomplete_hold_window')
continue
part = slice(end - window, end)
normal = trace['normal'][part, 0, 2]
gap = trace['gap'][part, 0]
weight = trace['weight'][part, 0]
row = {'segment': index, 'skill': name, 'ticks': end-start,
'checked_hold_dwell_s': window*.002, 'normal_min_N': float(normal.min()),
'normal_max_N': float(normal.max()), 'gap_min_m': float(gap.min())}
if kind in (0, 3):
contact_fraction = float((normal >= .5 * weight).float().mean())
require(contact_fraction >= .8, f'{index}:{name}:support_contact_fraction')
require(bool(((normal >= .5 * weight) & (gap <= .002)).all()), f'{index}:{name}:continuous_support_dwell_and_hold')
require(float(normal.mean()) >= .5 * float(weight.mean()), f'{index}:{name}:support_mean_load')
row['contact_fraction'] = contact_fraction
if kind == 3:
error = (trace['foot'][part] - trace['target'][part]).norm(dim=-1)
require(error.max() <= .05, f'{index}:landing_error')
row['landing_error_max_m'] = float(error.max())
else:
require(bool((gap >= .01).all() and (normal <= 2.).all()), f'{index}:{name}:air_hold')
if kind == 1:
unload = (trace['age_s'][start:end, 0] >= .9) & (trace['age_s'][start:end, 0] <= 1.)
first_normal = initial['normal'][0, 2] if start == 0 else trace['normal'][start-1, 0, 2]
remain = trace['normal'][start:end, 0, 2][unload]
require(len(remain) and remain.mean() <= .8 * first_normal, f'{index}:unload')
else:
command = torch.tensor(sequence['direction_m']) * (1 if swing_index % 2 == 0 else -1)
entry = trace['foot'][start-1, 0, :2]
displacement = trace['foot'][end-1, 0, :2] - entry
directed = (trace['foot'][part, 0, :2] - entry) * command.sign()
require((directed >= .5 * command.abs()).all(), f'{index}:continuous_signed_swing_hold')
require(((displacement * command.sign()) >= .5 * command.abs()).all(), f'{index}:signed_swing')
row['actual_xy_m'] = displacement.tolist()
row['command_xy_m'] = command.tolist()
swing_index += 1
windows.append(row)
# Reconstruct the exact private disturbance schedule, then compare to the
# original applied wrench tensor at each pre-step clock (not reward labels).
schedule_doc = json.loads((run / 'force-schedule.json').read_text())
spec = importlib.util.spec_from_file_location('four_force_audit', ROOT / 'source/SkillFactory/SkillFactory/legs/four_skills/forces.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
row = schedule_doc['rows'][0]
require(schedule_doc['seed'] == approval['seed'], 'schedule_seed_not_approved')
require(row['tier'] == approval['tier'], 'schedule_tier_not_approved')
require(row['env_id'] == 0 and row['episode_id'] == int(initial['episode_id'][0]), 'schedule_episode_not_initial_state')
runtime = json.loads((run / 'runtime-identity.json').read_text())
payload_index = runtime['payload_body_index']
require(len(runtime['body_names']) == trace['external'].shape[-2], 'body_index_layout_mismatch')
require(runtime['body_names'][payload_index] == 'virtual_payload', 'wrong_external_force_body')
schedule = module.ForceSchedule(schedule_doc['seed'], [row['env_id']], row['tier'], 'cpu',
duration_s=schedule_doc['duration_s'], episode_ids=[row['episode_id']])
require(schedule.manifest() == schedule_doc, 'force_manifest_parameters_changed')
times = torch.cat((initial['time_s'][:1], trace['time_s'][:-1, 0]))
# CPU batched audit is in chunks, preserving every original sample.
maximum_force_difference = 0.
for start in range(0, count, 2048):
end = min(count, start + 2048)
predicted = torch.stack([schedule.at(t.reshape(1))[0] for t in times[start:end]])
actual = trace['external'][start:end, 0]
expected = torch.zeros_like(actual)
expected[:, payload_index, :3] = predicted
maximum_force_difference = max(maximum_force_difference, float((expected - actual).abs().max()))
require(maximum_force_difference <= 2e-6, 'sampled_force_not_actually_applied')
tier_cap = config['force']['tiers_N'][approval['tier']]
require(trace['external'][..., :3].norm(dim=-1).max() <= tier_cap + .000001, 'force_out_of_approved_tier_range')
require(supervisor['success'] and supervisor['owned_cleanup'] and supervisor['waited_exit'], 'supervision_or_cleanup')
require(worker.get('functional_success') and worker.get('completed_loops') == loops, 'incomplete_original_result')
return {'passed': not reasons, 'failure_reasons': reasons, 'run': str(run.resolve()),
'physics_ticks': count, 'requested_loops': loops, 'windows': windows,
'max_force_reconstruction_difference_N': maximum_force_difference,
'scope': 'CPU physical-tensor audit; runtime continuity separately requires original hooks/supervisor'}
def compare_pair(analytic, trained):
a, b = Path(analytic), Path(trained)
aa = json.loads((a / 'approval.json').read_text())
ab = json.loads((b / 'approval.json').read_text())
conditions = {k: aa[k] == ab[k] for k in ('seed', 'sequence', 'loops', 'tier', 'config_sha256', 'mode')}
identity_file = b / 'checkpoint-identity.json'
trained_identity = json.loads(identity_file.read_text()) if identity_file.exists() else {}
controllers = (aa['controller'] == 'analytic' and aa['checkpoint'] is None
and ab['controller'] == 'trained' and bool(ab['checkpoint_sha256'])
and trained_identity.get('sha256') == ab['checkpoint_sha256']
and type(trained_identity.get('update')) is int and trained_identity['update'] > 0
and trained_identity.get('identity', {}).get('config_sha256') == ab['config_sha256'])
integrity = {name: json.loads((p / 'supervisor-result.json').read_text())['success'] for name, p in [('analytic', a), ('trained', b)]}
sa = json.loads((a / 'force-schedule.json').read_text())
sb = json.loads((b / 'force-schedule.json').read_text())
ia = torch.load(a / 'initial-state.pt', weights_only=True, map_location='cpu')
ib = torch.load(b / 'initial-state.pt', weights_only=True, map_location='cpu')
fields = ['root', 'joint_pos', 'joint_vel', 'normal', 'foot', 'target', 'reference', 'q_ik', 'actor_observation']
equality = {k: torch.equal(ia[k], ib[k]) for k in fields}
ta = torch.load(a / 'trace.pt', weights_only=True, map_location='cpu')
tb = torch.load(b / 'trace.pt', weights_only=True, map_location='cpu')
shared = min(len(ta['tick']), len(tb['tick']))
force_prefix_equal = torch.equal(ta['external'][:shared], tb['external'][:shared])
return {'paired': sa == sb and all(equality.values()) and all(conditions.values()) and controllers and all(integrity.values()) and force_prefix_equal,
'approved_conditions_equal': conditions, 'analytic_vs_frozen_trained': controllers,
'runtime_integrity': integrity, 'common_prefix_ticks': shared, 'applied_force_common_prefix_equal': force_prefix_equal,
'analytic_physical_audit_passed': audit(a)['passed'], 'trained_physical_audit_passed': audit(b)['passed'],
'disturbance_schedule_equal': sa == sb,
'observed_initial_state_equal': equality,
'limitation': 'observed physical/controller state, not a proof of complete hidden PhysX state identity'}
if __name__ == '__main__':
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('run', type=Path)
p.add_argument('--pair', type=Path)
p.add_argument('--output', type=Path)
args = p.parse_args()
result = compare_pair(args.run, args.pair) if args.pair else audit(args.run)
text = json.dumps(result, indent=2, allow_nan=False) + '\n'
if args.output:
args.output.write_text(text)
print(text)
raise SystemExit(0 if result.get('passed', result.get('paired', False)) else 1)