Files
SkillFactory/scripts/verify_cycle_sim_stress.py
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

344 lines
19 KiB
Python

#!/usr/bin/env python3
"""One supervised, bounded, non-learning nominal residual matrix. No model construction."""
import argparse
import fcntl
import hashlib
import io
import json
import os
import subprocess
import sys
import time
import traceback
from pathlib import Path
from runtime_common import finish_run, write_json
ROOT = Path(__file__).resolve().parents[1]
WINDOW_ROOT = ROOT / 'logs/contact_cycle/autonomous_sim_resume_v1'
WINDOW = WINDOW_ROOT / 'window.json'
WINDOW_SHA256 = '511d3f98a2c9ec7b78b1cb7f8f72e28ef06de6dcf96ecf2493ac7399d1eefbc8'
OUTPUT_ROOT = WINDOW_ROOT / 'phase01_nominal_residual'
PLAN = ROOT / 'configs/contact_cycle/simulation_stress_v1.json'
def sha256(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
class WindowGuard:
"""Same-boot absolute deadline; wall UTC never authorizes a launch or control."""
def __init__(self):
if sha256(WINDOW) != WINDOW_SHA256:
raise RuntimeError('Window contract hash changed')
self.window = json.loads(WINDOW.read_text())
self.last_disk_check = -float('inf')
self.bytes = 0
def check(self, reserve=0, force_disk=False):
now = time.clock_gettime(time.CLOCK_BOOTTIME)
boot = Path('/proc/sys/kernel/random/boot_id').read_text().strip()
remaining = self.window['deadline_boottime'] - now
if boot != self.window['boot_id'] or remaining <= self.window['cleanup_margin_s']:
raise RuntimeError('Boot/deadline guard tripped; no additional physics authorized')
if force_disk or reserve or now - self.last_disk_check >= 1:
self.bytes = sum(p.stat().st_size for p in WINDOW_ROOT.rglob('*') if p.is_file())
self.last_disk_check = now
# Reserve space for supervisor logs and final shutdown/error reports.
if self.bytes + reserve >= self.window['max_new_artifact_bytes'] - 8 * 1024**2:
raise RuntimeError('Artifact guard tripped')
return remaining
def verify_hashes(identity):
changed = [name for name, digest in identity.items() if sha256(ROOT / name) != digest]
if changed:
raise RuntimeError(f'Protected/registered inputs changed: {changed}')
def capture_identity():
protected = json.loads((WINDOW_ROOT / 'protected_inputs_before.json').read_text())
verify_hashes(protected)
files = {ROOT / name for name in protected}
files.update(p for p in (ROOT / 'source/SkillFactory').rglob('*')
if p.is_file() and '__pycache__' not in p.parts and p.suffix != '.pyc')
support = json.loads((ROOT / 'configs/standing/go2_fl_support_v0.json').read_text())
for key in ('source_bundle', 'support_asset'):
files.update(p for p in (ROOT / support[key]).rglob('*') if p.is_file())
files.update([Path(__file__), PLAN, WINDOW, WINDOW_ROOT / 'protected_inputs_before.json',
ROOT / 'scripts/runtime_common.py', ROOT / 'scripts/supervise_sim.py',
ROOT / 'tests/test_contact_cycle_sim_stress.py', ROOT / 'docs/contact_cycle_sim_stress.md'])
return {str(p.relative_to(ROOT)): sha256(p) for p in sorted(files)}
def assert_no_kit():
for p in Path('/proc').glob('[0-9]*/cmdline'):
try:
args = p.read_bytes().split(b'\0')
except (FileNotFoundError, PermissionError, ProcessLookupError):
continue
# Match executable/launcher tokens, not incidental substrings such as polkit.
if args and (Path(os.fsdecode(args[0])).name in ('kit', 'isaac-sim', 'isaac-sim.sh')
or any(a.endswith(b'/isaaclab.sh') for a in args)):
raise RuntimeError(f'Unrelated simulator process conflict: {p.parent.name}')
def supervise(output):
guard = WindowGuard()
guard.check(force_disk=True)
OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
with (OUTPUT_ROOT / 'writer.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
assert_no_kit()
output.mkdir(parents=True, exist_ok=False)
identity = capture_identity()
registration = {'window_sha256': WINDOW_SHA256, 'registered_boottime': time.clock_gettime(time.CLOCK_BOOTTIME),
'plan_sha256': sha256(PLAN), 'plan': json.loads(PLAN.read_text()), 'identity': identity,
'omissions': [], 'optimizer_steps': 0, 'policy_models_constructed': 0}
write_json(output / 'preregistration.json', registration)
remaining = guard.check(force_disk=True) # immediately before the sole Kit launch
timeout = min(2400, int(remaining - guard.window['cleanup_margin_s'] - 10))
if timeout <= 0:
raise RuntimeError('No safe launch time remains')
command = [sys.executable, str(ROOT / 'scripts/supervise_sim.py'),
'--log', str(output / 'simulator.log'), '--report', str(output / 'supervisor.json'),
'--timeout', str(timeout), '--', '/home/cen/IsaacLab/isaaclab.sh', '-p',
str(Path(__file__).resolve()), '--output', str(output), '--headless']
write_json(output / 'launch.json', {'command': command, 'remaining_s': remaining, 'timeout_s': timeout,
'preregistration_sha256': sha256(output / 'preregistration.json')})
environment = dict(os.environ, PYTHONPATH=str(ROOT / 'source/SkillFactory'), PYTHONUNBUFFERED='1')
environment.pop('CONDA_PREFIX', None)
environment.pop('VIRTUAL_ENV', None)
completed = subprocess.run(command, env=environment, check=False)
verify_hashes(identity)
if sha256(output / 'preregistration.json') != json.loads((output / 'launch.json').read_text())[
'preregistration_sha256']:
raise RuntimeError('Preregistration changed during the run')
write_json(output / 'integrity_after.json', {'unchanged': True, 'files_checked': len(identity),
'remaining_s': guard.check(force_disk=True),
'window_artifact_bytes': guard.bytes})
return completed.returncode
def claim_execution(output):
"""Consume an attempt once, before Kit; never overwrite completed/partial evidence."""
if (output / 'result.json').exists() or any(path.is_dir() for path in output.iterdir()):
raise FileExistsError('Attempt already contains runtime evidence; use a fresh output')
report = output / 'supervisor.json'
if report.exists():
try:
supervisor = json.loads(report.read_text())
except (ValueError, UnicodeError) as error:
raise FileExistsError('Existing supervisor evidence is malformed; preserve attempt') from error
# The unchanged supervisor writes precisely this initial report BEFORE spawning us.
initial_running = (isinstance(supervisor, dict) and set(supervisor) == {'passed', 'status', 'command'}
and supervisor['passed'] is False and supervisor['status'] == 'running'
and isinstance(supervisor['command'], list) and bool(supervisor['command'])
and all(isinstance(arg, str) for arg in supervisor['command']))
if not initial_running:
raise FileExistsError('Existing supervisor report is not a fresh running report')
# O_EXCL also prevents simultaneous internal invocations or retries after startup failure.
with (output / 'execution_started.json').open('x') as stream:
json.dump({'pid': os.getpid(), 'started_boottime': time.clock_gettime(time.CLOCK_BOOTTIME)}, stream)
def save_tensor(path, value, guard):
import torch
stream = io.BytesIO()
torch.save(value, stream)
guard.check(reserve=stream.tell())
with path.open('xb') as out:
out.write(stream.getbuffer())
def summarize_episode(episode, e, recorder, history, plan, state, functional):
import torch
from audit_contact_cycle_functional import evaluate
from SkillFactory.legs.contact_cycle.task import FAILURE_NAMES
from SkillFactory.legs.contact_cycle_sim.stress import first_failure
row = evaluate(episode, functional, 'analytic_reference')
row.update(env=e, controller_kind='analytic_reference_with_scripted_residual',
first_episode_samples=len(episode['time']),
phase_sample_counts={str(p): int((episode['phase'] == p).sum()) for p in range(6)},
first_failure=first_failure(episode, FAILURE_NAMES),
control_boundary_stop_s=float(episode['time'][-1, 0]) if recorder.stops[e] is not None else None,
early_stop=bool(recorder.early[e]))
stop = (recorder.stops[e] or len(recorder.frames)) // 5
h = {k: v[:stop, e:e+1].clone().contiguous() for k, v in history.items()}
row['terminated'] = bool(h['terminated'].any())
row['truncated'] = bool(h['truncated'].any())
row['termination_physics_tick'] = len(episode['time']) if row['terminated'] or row['truncated'] else None
row['termination_causes'] = ([name for j, name in enumerate(FAILURE_NAMES)
if episode['failure_flags'][-1, 0, j]] if row['terminated'] else
['timeout'] if row['truncated'] else [])
target = episode['processed_target']
nominal = state.nominal.cpu()
scale = state.env.cfg.support_parameters['action_scale_rad']
delta = target - nominal - scale * episode['reference_used_normalized']
row['max_actual_residual_rad'] = float(delta.abs().max())
row['max_requested_residual_rad'] = float(episode['residual_request_normalized'].abs().max()) * .05
row['max_absolute_target_offset_rad'] = float((target - nominal).abs().max())
row['checks'] = {
'actual_delta_bounded': row['max_actual_residual_rad'] <= plan['residual_limit_rad'] + 1e-6,
'applied_target_original_bounds': row['max_absolute_target_offset_rad'] <= scale + 1e-6,
'effective_delta_matches_target': bool(torch.allclose(delta, episode['effective_residual_rad'],
rtol=0, atol=1e-6)),
'finite_reward_action_history': all(bool(torch.isfinite(v).all()) for v in h.values()),
'finite_physics_trace': all(bool(torch.isfinite(v).all()) for v in episode.values()),
'nominal_no_external_disturbance': not bool(episode['scenario'].any() or episode['external'].any()),
'no_actor_metadata': not bool(episode['actor_metadata_available'].any()),
'termination_has_physics_cause': not row['terminated'] or row['first_failure'] is not None,
}
row['success'] = (row['sim_functional_passed'] and not row['early_stop']
and not row['terminated'] and all(row['checks'].values()))
return row, h
def run_case(env, case, plan, output, guard):
import torch
from SkillFactory.legs.contact_cycle_sim.stress import gate_residual, residual_signals
from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit
from SkillFactory.tasks.manager_based.contact_cycle_sim.mdp import state
from SkillFactory.tasks.manager_based.contact_cycle_sim.recording import SimCycleRecorder
guard.check(force_disk=True)
output.mkdir(exist_ok=False)
env.reset(seed=plan['environment_seed'])
s = state(env)
if s.robot.is_fixed_base or env.cfg.terminations.failed is None or bool(s.scenarios.any()):
raise RuntimeError('Require floating root, original failure termination and nominal scenarios')
requests = residual_signals(plan, case).to(env.device)
histories = {k: [] for k in ('reward', 'request', 'action', 'previous_action', 'terminated', 'truncated')}
thresholds = json.loads((ROOT / 'configs/contact_cycle/acceptance_v1.json').read_text())['gates']
with PhysicalSafetyAudit(env, thresholds) as safety, SimCycleRecorder(env) as recorder:
save_tensor(output / 'initial.pt', recorder.initial_state, guard)
for k in range(plan['controls_per_case']):
guard.check()
request = requests[k]
if case['kind'] == 'gated_sine':
request = gate_residual(request, s.reference()[0], plan['gated_phases'])
recorder.before_control(None, request)
_, reward, terminated, truncated, _ = env.step(request)
recorder.end_control(terminated, truncated, k == plan['controls_per_case'] - 1)
for name, value in zip(histories, (reward, request, env.action_manager.action,
env.action_manager.prev_action, terminated, truncated)):
histories[name].append(value.detach().cpu().clone())
safety.check()
if (k + 1) % 100 == 0:
print(f"{case['id']}: {k + 1}/900 controls", flush=True)
history = {k: torch.stack(v) for k, v in histories.items()}
trace = {k: torch.stack([f[k] for f in recorder.frames]) for k in recorder.frames[0]}
functional = json.loads((ROOT / plan['functional_config']).read_text())
rows, first_histories = [], {}
for e, stop in enumerate(recorder.stops):
episode = {k: v[:stop, e:e+1] for k, v in trace.items()}
row, h = summarize_episode(episode, e, recorder, history, plan, s, functional)
rows.append(row)
first_histories[str(e)] = h
selected = [r['env'] for r in rows if not r['success']][:4]
complete = [r['env'] for r in rows if r['first_episode_samples'] == 4500 and not r['early_stop']]
successful = [r['env'] for r in rows if r['success']]
representative = (successful or complete)[:1]
selected = sorted(set(selected + representative))
subset = {str(e): {k: v[:recorder.stops[e], e:e+1].clone().contiguous() for k, v in trace.items()}
for e in selected}
save_tensor(output / 'first_episode_traces.pt', subset, guard)
save_tensor(output / 'first_episode_control_histories.pt', first_histories, guard)
result = {'case': case, 'rows': rows, 'success_count': sum(r['success'] for r in rows),
'environment_count': env.num_envs, 'saved_trace_environments': selected,
'physics_safety': safety.report(),
'harness_verified': all(all(r['checks'].values()) for r in rows),
'force_provenance': {'normal': 'PhysX foot_contact net normal force, world frame',
'external': 'declared event wrench buffers; nominal all zero',
'guide': 'declared auxiliary fixture, no vertical guide force',
'auxiliary': 'declared phase-dependent vertical support with r cross F moments'},
'remaining_window_s': guard.check(force_disk=True)}
write_json(output / 'result.json', result)
print(f"CASE_RESULT {case['id']}: {result['success_count']}/8", flush=True)
return result
def run_matrix(env, output, guard, registration):
results = []
plan = registration['plan']
for case in plan['cases']:
verify_hashes(registration['identity'])
result = run_case(env, case, plan, output / case['id'], guard)
results.append(result)
write_json(output / 'matrix_progress.json', {
'completed_cases': [{'id': r['case']['id'], 'success_count': r['success_count']} for r in results],
'planned_case_count': len(plan['cases']), 'remaining_window_s': guard.check()})
baseline = all(r['success_count'] == 8 for r in results if r['case']['kind'] == 'zero')
verified = all(r['harness_verified'] for r in results)
stress = [r for r in results if r['case']['kind'] != 'zero']
return {'passed': verified and baseline, 'passed_means': 'harness verified AND both baselines; NOT stress success',
'harness_verified': verified, 'baseline_passed': baseline,
'case_count': len(results), 'environment_episodes': 8 * len(results),
'stress_case_count': len(stress), 'stress_full_success_cases': sum(r['success_count'] == 8 for r in stress),
'stress_success_episodes': sum(r['success_count'] for r in stress),
'case_counts': {r['case']['id']: r['success_count'] for r in results},
'controller_kind': 'analytic_reference_with_scripted_residual', 'accepted_skill': False,
'trained_residual_policy': False, 'optimizer_steps': 0, 'policy_models_constructed': 0,
'remaining_window_s': guard.check(force_disk=True),
'limitations': ['Nominal development-only deterministic environment; no external disturbances.',
'Gated lift/swing domain is not the all-phase domain.',
'No learned-policy, robustness, recovery, hardware or release claim.',
'First failure onset is 500Hz; actual environment termination/reset is control-rate.',
'First four failures plus representative complete traces; all eight rows/histories.']}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--supervise', action='store_true')
known, _ = parser.parse_known_args()
output = known.output.resolve()
if output.parent != OUTPUT_ROOT:
parser.error(f'Output must be a fresh attempt directory directly inside {OUTPUT_ROOT}')
if known.supervise:
raise SystemExit(supervise(output))
guard = WindowGuard()
guard.check(force_disk=True)
registration = json.loads((output / 'preregistration.json').read_text())
launch = json.loads((output / 'launch.json').read_text())
if (sha256(output / 'preregistration.json') != launch['preregistration_sha256']
or sha256(PLAN) != registration['plan_sha256']):
raise RuntimeError('Preregistration changed before launch')
verify_hashes(registration['identity'])
guard.check(reserve=1024)
claim_execution(output)
from isaaclab.app import AppLauncher
AppLauncher.add_app_launcher_args(parser)
args = parser.parse_args()
guard.check() # last pre-Kit authorization check
app = AppLauncher(args).app
env = None
result = {'passed': False, 'harness_verified': False, 'baseline_passed': False, 'optimizer_steps': 0}
code = 1
try:
from SkillFactory.tasks.manager_based.contact_cycle_sim.cfg import SimContactCycleEnvCfg
from SkillFactory.tasks.manager_based.contact_cycle_sim.env import SimContactCycleEnv
cfg = SimContactCycleEnvCfg()
cfg.scene.num_envs = registration['plan']['num_envs']
cfg.sim.device = args.device
cfg.seed = registration['plan']['environment_seed']
guard.check()
env = SimContactCycleEnv(cfg=cfg)
result = run_matrix(env, output, guard, registration)
verify_hashes(registration['identity'])
code = 0 if result['passed'] else 2
except BaseException:
result.update(passed=False, harness_verified=False, error=traceback.format_exc())
traceback.print_exc()
finally:
finish_run(app, env, output / 'result.json', result, code)
if __name__ == '__main__':
main()