9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
344 lines
19 KiB
Python
344 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""One supervised, bounded, non-learning nominal residual matrix. No model construction."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
from runtime_common import finish_run, write_json
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
WINDOW_ROOT = ROOT / 'logs/contact_cycle/autonomous_sim_resume_v1'
|
|
WINDOW = WINDOW_ROOT / 'window.json'
|
|
WINDOW_SHA256 = '511d3f98a2c9ec7b78b1cb7f8f72e28ef06de6dcf96ecf2493ac7399d1eefbc8'
|
|
OUTPUT_ROOT = WINDOW_ROOT / 'phase01_nominal_residual'
|
|
PLAN = ROOT / 'configs/contact_cycle/simulation_stress_v1.json'
|
|
|
|
|
|
def sha256(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
class WindowGuard:
|
|
"""Same-boot absolute deadline; wall UTC never authorizes a launch or control."""
|
|
|
|
def __init__(self):
|
|
if sha256(WINDOW) != WINDOW_SHA256:
|
|
raise RuntimeError('Window contract hash changed')
|
|
self.window = json.loads(WINDOW.read_text())
|
|
self.last_disk_check = -float('inf')
|
|
self.bytes = 0
|
|
|
|
def check(self, reserve=0, force_disk=False):
|
|
now = time.clock_gettime(time.CLOCK_BOOTTIME)
|
|
boot = Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
|
remaining = self.window['deadline_boottime'] - now
|
|
if boot != self.window['boot_id'] or remaining <= self.window['cleanup_margin_s']:
|
|
raise RuntimeError('Boot/deadline guard tripped; no additional physics authorized')
|
|
if force_disk or reserve or now - self.last_disk_check >= 1:
|
|
self.bytes = sum(p.stat().st_size for p in WINDOW_ROOT.rglob('*') if p.is_file())
|
|
self.last_disk_check = now
|
|
# Reserve space for supervisor logs and final shutdown/error reports.
|
|
if self.bytes + reserve >= self.window['max_new_artifact_bytes'] - 8 * 1024**2:
|
|
raise RuntimeError('Artifact guard tripped')
|
|
return remaining
|
|
|
|
|
|
def verify_hashes(identity):
|
|
changed = [name for name, digest in identity.items() if sha256(ROOT / name) != digest]
|
|
if changed:
|
|
raise RuntimeError(f'Protected/registered inputs changed: {changed}')
|
|
|
|
|
|
def capture_identity():
|
|
protected = json.loads((WINDOW_ROOT / 'protected_inputs_before.json').read_text())
|
|
verify_hashes(protected)
|
|
files = {ROOT / name for name in protected}
|
|
files.update(p for p in (ROOT / 'source/SkillFactory').rglob('*')
|
|
if p.is_file() and '__pycache__' not in p.parts and p.suffix != '.pyc')
|
|
support = json.loads((ROOT / 'configs/standing/go2_fl_support_v0.json').read_text())
|
|
for key in ('source_bundle', 'support_asset'):
|
|
files.update(p for p in (ROOT / support[key]).rglob('*') if p.is_file())
|
|
files.update([Path(__file__), PLAN, WINDOW, WINDOW_ROOT / 'protected_inputs_before.json',
|
|
ROOT / 'scripts/runtime_common.py', ROOT / 'scripts/supervise_sim.py',
|
|
ROOT / 'tests/test_contact_cycle_sim_stress.py', ROOT / 'docs/contact_cycle_sim_stress.md'])
|
|
return {str(p.relative_to(ROOT)): sha256(p) for p in sorted(files)}
|
|
|
|
|
|
def assert_no_kit():
|
|
for p in Path('/proc').glob('[0-9]*/cmdline'):
|
|
try:
|
|
args = p.read_bytes().split(b'\0')
|
|
except (FileNotFoundError, PermissionError, ProcessLookupError):
|
|
continue
|
|
# Match executable/launcher tokens, not incidental substrings such as polkit.
|
|
if args and (Path(os.fsdecode(args[0])).name in ('kit', 'isaac-sim', 'isaac-sim.sh')
|
|
or any(a.endswith(b'/isaaclab.sh') for a in args)):
|
|
raise RuntimeError(f'Unrelated simulator process conflict: {p.parent.name}')
|
|
|
|
|
|
def supervise(output):
|
|
guard = WindowGuard()
|
|
guard.check(force_disk=True)
|
|
OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
|
|
with (OUTPUT_ROOT / 'writer.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
assert_no_kit()
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
identity = capture_identity()
|
|
registration = {'window_sha256': WINDOW_SHA256, 'registered_boottime': time.clock_gettime(time.CLOCK_BOOTTIME),
|
|
'plan_sha256': sha256(PLAN), 'plan': json.loads(PLAN.read_text()), 'identity': identity,
|
|
'omissions': [], 'optimizer_steps': 0, 'policy_models_constructed': 0}
|
|
write_json(output / 'preregistration.json', registration)
|
|
remaining = guard.check(force_disk=True) # immediately before the sole Kit launch
|
|
timeout = min(2400, int(remaining - guard.window['cleanup_margin_s'] - 10))
|
|
if timeout <= 0:
|
|
raise RuntimeError('No safe launch time remains')
|
|
command = [sys.executable, str(ROOT / 'scripts/supervise_sim.py'),
|
|
'--log', str(output / 'simulator.log'), '--report', str(output / 'supervisor.json'),
|
|
'--timeout', str(timeout), '--', '/home/cen/IsaacLab/isaaclab.sh', '-p',
|
|
str(Path(__file__).resolve()), '--output', str(output), '--headless']
|
|
write_json(output / 'launch.json', {'command': command, 'remaining_s': remaining, 'timeout_s': timeout,
|
|
'preregistration_sha256': sha256(output / 'preregistration.json')})
|
|
environment = dict(os.environ, PYTHONPATH=str(ROOT / 'source/SkillFactory'), PYTHONUNBUFFERED='1')
|
|
environment.pop('CONDA_PREFIX', None)
|
|
environment.pop('VIRTUAL_ENV', None)
|
|
completed = subprocess.run(command, env=environment, check=False)
|
|
verify_hashes(identity)
|
|
if sha256(output / 'preregistration.json') != json.loads((output / 'launch.json').read_text())[
|
|
'preregistration_sha256']:
|
|
raise RuntimeError('Preregistration changed during the run')
|
|
write_json(output / 'integrity_after.json', {'unchanged': True, 'files_checked': len(identity),
|
|
'remaining_s': guard.check(force_disk=True),
|
|
'window_artifact_bytes': guard.bytes})
|
|
return completed.returncode
|
|
|
|
|
|
def claim_execution(output):
|
|
"""Consume an attempt once, before Kit; never overwrite completed/partial evidence."""
|
|
if (output / 'result.json').exists() or any(path.is_dir() for path in output.iterdir()):
|
|
raise FileExistsError('Attempt already contains runtime evidence; use a fresh output')
|
|
report = output / 'supervisor.json'
|
|
if report.exists():
|
|
try:
|
|
supervisor = json.loads(report.read_text())
|
|
except (ValueError, UnicodeError) as error:
|
|
raise FileExistsError('Existing supervisor evidence is malformed; preserve attempt') from error
|
|
# The unchanged supervisor writes precisely this initial report BEFORE spawning us.
|
|
initial_running = (isinstance(supervisor, dict) and set(supervisor) == {'passed', 'status', 'command'}
|
|
and supervisor['passed'] is False and supervisor['status'] == 'running'
|
|
and isinstance(supervisor['command'], list) and bool(supervisor['command'])
|
|
and all(isinstance(arg, str) for arg in supervisor['command']))
|
|
if not initial_running:
|
|
raise FileExistsError('Existing supervisor report is not a fresh running report')
|
|
# O_EXCL also prevents simultaneous internal invocations or retries after startup failure.
|
|
with (output / 'execution_started.json').open('x') as stream:
|
|
json.dump({'pid': os.getpid(), 'started_boottime': time.clock_gettime(time.CLOCK_BOOTTIME)}, stream)
|
|
|
|
|
|
def save_tensor(path, value, guard):
|
|
import torch
|
|
|
|
stream = io.BytesIO()
|
|
torch.save(value, stream)
|
|
guard.check(reserve=stream.tell())
|
|
with path.open('xb') as out:
|
|
out.write(stream.getbuffer())
|
|
|
|
|
|
def summarize_episode(episode, e, recorder, history, plan, state, functional):
|
|
import torch
|
|
from audit_contact_cycle_functional import evaluate
|
|
from SkillFactory.legs.contact_cycle.task import FAILURE_NAMES
|
|
from SkillFactory.legs.contact_cycle_sim.stress import first_failure
|
|
|
|
row = evaluate(episode, functional, 'analytic_reference')
|
|
row.update(env=e, controller_kind='analytic_reference_with_scripted_residual',
|
|
first_episode_samples=len(episode['time']),
|
|
phase_sample_counts={str(p): int((episode['phase'] == p).sum()) for p in range(6)},
|
|
first_failure=first_failure(episode, FAILURE_NAMES),
|
|
control_boundary_stop_s=float(episode['time'][-1, 0]) if recorder.stops[e] is not None else None,
|
|
early_stop=bool(recorder.early[e]))
|
|
stop = (recorder.stops[e] or len(recorder.frames)) // 5
|
|
h = {k: v[:stop, e:e+1].clone().contiguous() for k, v in history.items()}
|
|
row['terminated'] = bool(h['terminated'].any())
|
|
row['truncated'] = bool(h['truncated'].any())
|
|
row['termination_physics_tick'] = len(episode['time']) if row['terminated'] or row['truncated'] else None
|
|
row['termination_causes'] = ([name for j, name in enumerate(FAILURE_NAMES)
|
|
if episode['failure_flags'][-1, 0, j]] if row['terminated'] else
|
|
['timeout'] if row['truncated'] else [])
|
|
target = episode['processed_target']
|
|
nominal = state.nominal.cpu()
|
|
scale = state.env.cfg.support_parameters['action_scale_rad']
|
|
delta = target - nominal - scale * episode['reference_used_normalized']
|
|
row['max_actual_residual_rad'] = float(delta.abs().max())
|
|
row['max_requested_residual_rad'] = float(episode['residual_request_normalized'].abs().max()) * .05
|
|
row['max_absolute_target_offset_rad'] = float((target - nominal).abs().max())
|
|
row['checks'] = {
|
|
'actual_delta_bounded': row['max_actual_residual_rad'] <= plan['residual_limit_rad'] + 1e-6,
|
|
'applied_target_original_bounds': row['max_absolute_target_offset_rad'] <= scale + 1e-6,
|
|
'effective_delta_matches_target': bool(torch.allclose(delta, episode['effective_residual_rad'],
|
|
rtol=0, atol=1e-6)),
|
|
'finite_reward_action_history': all(bool(torch.isfinite(v).all()) for v in h.values()),
|
|
'finite_physics_trace': all(bool(torch.isfinite(v).all()) for v in episode.values()),
|
|
'nominal_no_external_disturbance': not bool(episode['scenario'].any() or episode['external'].any()),
|
|
'no_actor_metadata': not bool(episode['actor_metadata_available'].any()),
|
|
'termination_has_physics_cause': not row['terminated'] or row['first_failure'] is not None,
|
|
}
|
|
row['success'] = (row['sim_functional_passed'] and not row['early_stop']
|
|
and not row['terminated'] and all(row['checks'].values()))
|
|
return row, h
|
|
|
|
|
|
def run_case(env, case, plan, output, guard):
|
|
import torch
|
|
from SkillFactory.legs.contact_cycle_sim.stress import gate_residual, residual_signals
|
|
from SkillFactory.tasks.manager_based.contact_cycle.recording import PhysicalSafetyAudit
|
|
from SkillFactory.tasks.manager_based.contact_cycle_sim.mdp import state
|
|
from SkillFactory.tasks.manager_based.contact_cycle_sim.recording import SimCycleRecorder
|
|
|
|
guard.check(force_disk=True)
|
|
output.mkdir(exist_ok=False)
|
|
env.reset(seed=plan['environment_seed'])
|
|
s = state(env)
|
|
if s.robot.is_fixed_base or env.cfg.terminations.failed is None or bool(s.scenarios.any()):
|
|
raise RuntimeError('Require floating root, original failure termination and nominal scenarios')
|
|
requests = residual_signals(plan, case).to(env.device)
|
|
histories = {k: [] for k in ('reward', 'request', 'action', 'previous_action', 'terminated', 'truncated')}
|
|
thresholds = json.loads((ROOT / 'configs/contact_cycle/acceptance_v1.json').read_text())['gates']
|
|
with PhysicalSafetyAudit(env, thresholds) as safety, SimCycleRecorder(env) as recorder:
|
|
save_tensor(output / 'initial.pt', recorder.initial_state, guard)
|
|
for k in range(plan['controls_per_case']):
|
|
guard.check()
|
|
request = requests[k]
|
|
if case['kind'] == 'gated_sine':
|
|
request = gate_residual(request, s.reference()[0], plan['gated_phases'])
|
|
recorder.before_control(None, request)
|
|
_, reward, terminated, truncated, _ = env.step(request)
|
|
recorder.end_control(terminated, truncated, k == plan['controls_per_case'] - 1)
|
|
for name, value in zip(histories, (reward, request, env.action_manager.action,
|
|
env.action_manager.prev_action, terminated, truncated)):
|
|
histories[name].append(value.detach().cpu().clone())
|
|
safety.check()
|
|
if (k + 1) % 100 == 0:
|
|
print(f"{case['id']}: {k + 1}/900 controls", flush=True)
|
|
history = {k: torch.stack(v) for k, v in histories.items()}
|
|
trace = {k: torch.stack([f[k] for f in recorder.frames]) for k in recorder.frames[0]}
|
|
functional = json.loads((ROOT / plan['functional_config']).read_text())
|
|
rows, first_histories = [], {}
|
|
for e, stop in enumerate(recorder.stops):
|
|
episode = {k: v[:stop, e:e+1] for k, v in trace.items()}
|
|
row, h = summarize_episode(episode, e, recorder, history, plan, s, functional)
|
|
rows.append(row)
|
|
first_histories[str(e)] = h
|
|
selected = [r['env'] for r in rows if not r['success']][:4]
|
|
complete = [r['env'] for r in rows if r['first_episode_samples'] == 4500 and not r['early_stop']]
|
|
successful = [r['env'] for r in rows if r['success']]
|
|
representative = (successful or complete)[:1]
|
|
selected = sorted(set(selected + representative))
|
|
subset = {str(e): {k: v[:recorder.stops[e], e:e+1].clone().contiguous() for k, v in trace.items()}
|
|
for e in selected}
|
|
save_tensor(output / 'first_episode_traces.pt', subset, guard)
|
|
save_tensor(output / 'first_episode_control_histories.pt', first_histories, guard)
|
|
result = {'case': case, 'rows': rows, 'success_count': sum(r['success'] for r in rows),
|
|
'environment_count': env.num_envs, 'saved_trace_environments': selected,
|
|
'physics_safety': safety.report(),
|
|
'harness_verified': all(all(r['checks'].values()) for r in rows),
|
|
'force_provenance': {'normal': 'PhysX foot_contact net normal force, world frame',
|
|
'external': 'declared event wrench buffers; nominal all zero',
|
|
'guide': 'declared auxiliary fixture, no vertical guide force',
|
|
'auxiliary': 'declared phase-dependent vertical support with r cross F moments'},
|
|
'remaining_window_s': guard.check(force_disk=True)}
|
|
write_json(output / 'result.json', result)
|
|
print(f"CASE_RESULT {case['id']}: {result['success_count']}/8", flush=True)
|
|
return result
|
|
|
|
|
|
def run_matrix(env, output, guard, registration):
|
|
results = []
|
|
plan = registration['plan']
|
|
for case in plan['cases']:
|
|
verify_hashes(registration['identity'])
|
|
result = run_case(env, case, plan, output / case['id'], guard)
|
|
results.append(result)
|
|
write_json(output / 'matrix_progress.json', {
|
|
'completed_cases': [{'id': r['case']['id'], 'success_count': r['success_count']} for r in results],
|
|
'planned_case_count': len(plan['cases']), 'remaining_window_s': guard.check()})
|
|
baseline = all(r['success_count'] == 8 for r in results if r['case']['kind'] == 'zero')
|
|
verified = all(r['harness_verified'] for r in results)
|
|
stress = [r for r in results if r['case']['kind'] != 'zero']
|
|
return {'passed': verified and baseline, 'passed_means': 'harness verified AND both baselines; NOT stress success',
|
|
'harness_verified': verified, 'baseline_passed': baseline,
|
|
'case_count': len(results), 'environment_episodes': 8 * len(results),
|
|
'stress_case_count': len(stress), 'stress_full_success_cases': sum(r['success_count'] == 8 for r in stress),
|
|
'stress_success_episodes': sum(r['success_count'] for r in stress),
|
|
'case_counts': {r['case']['id']: r['success_count'] for r in results},
|
|
'controller_kind': 'analytic_reference_with_scripted_residual', 'accepted_skill': False,
|
|
'trained_residual_policy': False, 'optimizer_steps': 0, 'policy_models_constructed': 0,
|
|
'remaining_window_s': guard.check(force_disk=True),
|
|
'limitations': ['Nominal development-only deterministic environment; no external disturbances.',
|
|
'Gated lift/swing domain is not the all-phase domain.',
|
|
'No learned-policy, robustness, recovery, hardware or release claim.',
|
|
'First failure onset is 500Hz; actual environment termination/reset is control-rate.',
|
|
'First four failures plus representative complete traces; all eight rows/histories.']}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--supervise', action='store_true')
|
|
known, _ = parser.parse_known_args()
|
|
output = known.output.resolve()
|
|
if output.parent != OUTPUT_ROOT:
|
|
parser.error(f'Output must be a fresh attempt directory directly inside {OUTPUT_ROOT}')
|
|
if known.supervise:
|
|
raise SystemExit(supervise(output))
|
|
guard = WindowGuard()
|
|
guard.check(force_disk=True)
|
|
registration = json.loads((output / 'preregistration.json').read_text())
|
|
launch = json.loads((output / 'launch.json').read_text())
|
|
if (sha256(output / 'preregistration.json') != launch['preregistration_sha256']
|
|
or sha256(PLAN) != registration['plan_sha256']):
|
|
raise RuntimeError('Preregistration changed before launch')
|
|
verify_hashes(registration['identity'])
|
|
guard.check(reserve=1024)
|
|
claim_execution(output)
|
|
from isaaclab.app import AppLauncher
|
|
|
|
AppLauncher.add_app_launcher_args(parser)
|
|
args = parser.parse_args()
|
|
guard.check() # last pre-Kit authorization check
|
|
app = AppLauncher(args).app
|
|
env = None
|
|
result = {'passed': False, 'harness_verified': False, 'baseline_passed': False, 'optimizer_steps': 0}
|
|
code = 1
|
|
try:
|
|
from SkillFactory.tasks.manager_based.contact_cycle_sim.cfg import SimContactCycleEnvCfg
|
|
from SkillFactory.tasks.manager_based.contact_cycle_sim.env import SimContactCycleEnv
|
|
|
|
cfg = SimContactCycleEnvCfg()
|
|
cfg.scene.num_envs = registration['plan']['num_envs']
|
|
cfg.sim.device = args.device
|
|
cfg.seed = registration['plan']['environment_seed']
|
|
guard.check()
|
|
env = SimContactCycleEnv(cfg=cfg)
|
|
result = run_matrix(env, output, guard, registration)
|
|
verify_hashes(registration['identity'])
|
|
code = 0 if result['passed'] else 2
|
|
except BaseException:
|
|
result.update(passed=False, harness_verified=False, error=traceback.format_exc())
|
|
traceback.print_exc()
|
|
finally:
|
|
finish_run(app, env, output / 'result.json', result, code)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|