feat: checkpoint Go2 FL simulation performance v0.4.2
Include callable skill and live-GUI prerequisites, epoch sensor reads, packed snapshots, sampled bit-exact kinematics, and verified-tick rendering without changing the physical/control schedule. Record bounded analytic S1 GUI/headless qualification, original failures and CPU evidence (1557 passed, 6 skipped). Preserve accepted_skill=false and sub-real-time limitations. Exclude native authorities, caches, large media and local delivery copies; retain sealed artifact digest.
This commit is contained in:
@@ -0,0 +1,723 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Separately approved functional campaigns; optional frozen --followup-v3.
|
||||
|
||||
Default: PREP + reference/u256 1/3 cycles. Followup: PREP + u2563 + six paired
|
||||
three-cycle groups. No training/retry or old authority. --campaign is the sole
|
||||
supervisor; workers require its inherited authenticated FD before AppLauncher.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import functional_flow_launch_v2 as launch
|
||||
|
||||
ROOT, b = launch.ROOT, launch.b
|
||||
|
||||
|
||||
def actual_runtime(payload):
|
||||
"""Config-only setup shared by PREP and RUN. Never constructs an environment."""
|
||||
from isaaclab.app import AppLauncher
|
||||
native = Path(payload['native'])
|
||||
if payload.get('skill_v4'):
|
||||
import skill_launch_v4
|
||||
gui_enabled = skill_launch_v4.gui_enabled(payload)
|
||||
else:
|
||||
gui_enabled = False
|
||||
gui_args = (' --/app/window/title=Go2_FL_Contact_Cycle_0.4.1 --/app/window/width=1440'
|
||||
' --/app/window/height=960 --/app/window/x=120 --/app/window/y=80'
|
||||
' --/app/file/ignoreUnsavedOnExit=true') if gui_enabled else ''
|
||||
app = AppLauncher(headless=not gui_enabled, visualizer='kit' if gui_enabled else 'none',
|
||||
device='cuda:0', fast_shutdown=True,
|
||||
kit_args=f'--/log/file={native / "kit.log"}' + gui_args).app
|
||||
try:
|
||||
# Source inventory learned in PREP is independently rehashed before RUN.
|
||||
inventory = None if payload['mode'] == 'prepare' else launch.read(payload['inventory'], payload['inventory_sha256'])
|
||||
for name in (() if inventory is None else inventory['modules']):
|
||||
importlib.import_module(name)
|
||||
task = 'SkillFactory.tasks.manager_based.contact_cycle_sim.'
|
||||
leg = 'SkillFactory.legs.contact_cycle_sim.'
|
||||
modules = {name: importlib.import_module(task + name) for name in (
|
||||
'evaluation_cfg', 'evaluation_env', 'evaluation_mdp', 'evaluation_rng_scope',
|
||||
'evaluation_instrumentation', 'functional_flow_recording_v2')}
|
||||
modules.update({name: importlib.import_module(leg + name) for name in (
|
||||
'functional_flow_v2', 'functional_flow_binding_v2', 'evaluation_binding', 'replay')})
|
||||
if payload.get('followup_v3'):
|
||||
for name in ('functional_followup_v3', 'functional_followup_binding_v3'):
|
||||
modules[name] = importlib.import_module(leg + name)
|
||||
importlib.import_module('functional_gpu_watch_v3')
|
||||
if payload.get('skill_v4'):
|
||||
for name in ('skill_v4', 'skill_mailbox_v4', 'skill_binding_v4',
|
||||
'functional_followup_v3', 'functional_followup_binding_v3'):
|
||||
modules[name] = importlib.import_module(leg + name)
|
||||
modules['skill_recording_v4'] = importlib.import_module(task + 'skill_recording_v4')
|
||||
modules['performance_v042'] = importlib.import_module(task + 'performance_v042')
|
||||
modules['sensor_epoch_v042'] = importlib.import_module(task + 'sensor_epoch_v042')
|
||||
modules['packed_recording_v042'] = importlib.import_module(task + 'packed_recording_v042')
|
||||
modules['readonly_scope_v042'] = importlib.import_module(task + 'readonly_scope_v042')
|
||||
modules['batched_kinematics_v042'] = importlib.import_module(task + 'batched_kinematics_v042')
|
||||
modules['live_cadence_v042'] = importlib.import_module(task + 'live_cadence_v042')
|
||||
if payload['spec']['gui']:
|
||||
# PREP has no GUI experience, but must preload the exact finite
|
||||
# presentation dependency before inventory/binding (no viewport
|
||||
# construction or physics). RUN already enables this via Kit.
|
||||
import omni.kit.app
|
||||
manager = omni.kit.app.get_app().get_extension_manager()
|
||||
for extension in ('omni.ui', 'omni.kit.viewport.utility', 'omni.replicator.core'):
|
||||
if not manager.set_extension_enabled_immediate(extension, True):
|
||||
raise RuntimeError('gui_extension_unavailable:' + extension)
|
||||
modules['skill_gui_v041'] = importlib.import_module(task + 'skill_gui_v041')
|
||||
if payload['spec']['gui']:
|
||||
modules['skill_gui_v041'].prepare_native()
|
||||
# GUI experience preloads Replicator. Match that exact module
|
||||
# availability in headless PREP rather than relaxing its guard.
|
||||
importlib.import_module(launch.owner.OPTIONAL_REPLICATOR)
|
||||
for name in ('skill_launch_v4', 'functional_followup_launch_v3', 'functional_gpu_watch_v3',
|
||||
'evaluate_cycle_functional_flow'):
|
||||
importlib.import_module(name)
|
||||
binding_module = modules['skill_binding_v4' if payload.get('skill_v4') else
|
||||
'functional_followup_binding_v3' if payload.get('followup_v3') else 'functional_flow_binding_v2']
|
||||
binder = modules['evaluation_binding']
|
||||
for name in binder.REQUIRED_RUNTIME:
|
||||
importlib.import_module(name)
|
||||
instrument = modules['evaluation_instrumentation']
|
||||
for name in instrument.BACKEND_SOURCE_INVENTORY:
|
||||
importlib.import_module(name)
|
||||
instrument.verify_backend_sources()
|
||||
replicator = launch.owner.preimport_replicator()
|
||||
if inventory is not None:
|
||||
launch.owner.check_replicator_inventory(inventory)
|
||||
import torch
|
||||
torch.cuda.init()
|
||||
torch.cuda.get_rng_state(torch.device('cuda:0'))
|
||||
if torch.cuda.get_device_name(0) != 'NVIDIA GeForce RTX 5080':
|
||||
raise ValueError('Require approved RTX5080')
|
||||
cfg = modules['evaluation_cfg'].EvaluationContactCycleEnvCfg()
|
||||
cfg.sim.device, cfg.seed = 'cuda:0', 93501
|
||||
if payload.get('skill_v4') and payload['spec']['gui']:
|
||||
# Same explicit presentation config/imports in PREP and RUN. Only
|
||||
# RUN creates KitVisualizer/window; no physics config is changed.
|
||||
from isaaclab_visualizers.kit import KitVisualizerCfg
|
||||
importlib.import_module('isaaclab_visualizers.kit.kit_visualizer')
|
||||
# Observed GUI constructor-only camera helper imports must be in
|
||||
# PREP's exact provenance before construction, not waived afterward.
|
||||
importlib.import_module('isaaclab_physx.renderers.kit_viewport_utils')
|
||||
# Full GUI run 10: first render lazily loads this exact utility.
|
||||
importlib.import_module('isaaclab_physx.renderers.isaac_rtx_renderer_utils')
|
||||
cfg.sim.visualizer_cfgs = [KitVisualizerCfg()]
|
||||
if payload.get('skill_v4'):
|
||||
perf = modules['performance_v042']
|
||||
perf.RUNTIME_MATH_QUALIFICATION = None
|
||||
if perf.options(ROOT)['mode'] in ('combined', 'combined_display'):
|
||||
nominal = torch.tensor([j['q_default'] for j in cfg.leg_bundle['spec']['joints']], device='cuda:0')
|
||||
perf.RUNTIME_MATH_QUALIFICATION = modules['batched_kinematics_v042'].qualify(
|
||||
cfg.leg_bundle, cfg.residual_parameters['reference'], nominal, 'cuda:0')
|
||||
launch.owner.preforward_video_recorder(cfg, modules['evaluation_env'])
|
||||
constructor_imports = launch.owner.preimport_constructor_modules(binder)
|
||||
constructor_export, export_metadata = launch.owner.physx_constructor_export(binder, resolve=True)
|
||||
config_imports = launch.owner.preimport_config_references(cfg, binder)
|
||||
binder.normalize_config(cfg)
|
||||
rng_scope = modules['evaluation_rng_scope']
|
||||
rng_inventory = rng_scope.qualify_config(cfg)
|
||||
rng_inventory.update(optional_replicator=replicator, constructor_imports=constructor_imports,
|
||||
constructor_export=export_metadata, config_reference_imports=config_imports)
|
||||
instrument.qualify_backend_rng(rng_scope.backend_rng(None))
|
||||
return SimpleNamespace(app=app, modules=modules, cfg=cfg, torch=torch, binding_module=binding_module,
|
||||
constructor_export=constructor_export, rng_inventory=rng_inventory)
|
||||
except BaseException:
|
||||
traceback.print_exc()
|
||||
app.close(exit_code=1)
|
||||
raise
|
||||
|
||||
|
||||
def verify_runtime(runtime, expected, inventory):
|
||||
binder = runtime.modules['evaluation_binding']
|
||||
launch.owner.physx_constructor_export(binder, expected=runtime.constructor_export)
|
||||
launch.owner.check_replicator_inventory(inventory)
|
||||
return runtime.binding_module.verify(expected, ROOT, runtime.cfg, inventory)
|
||||
|
||||
|
||||
def worker(payload):
|
||||
root, output = Path(payload['root']), Path(payload['output'])
|
||||
is_followup = payload.get('followup_v3', False)
|
||||
is_skill = payload.get('skill_v4', False)
|
||||
if is_skill:
|
||||
import skill_launch_v4 as skill
|
||||
if is_followup:
|
||||
import functional_followup_launch_v3 as followup
|
||||
guard = launch.Guard(root, payload['start'], payload['deadline'],
|
||||
cap=payload.get('owned_cap', launch.CAP), stdout_slots=14 if is_followup else 5)
|
||||
interrupted = {sig: signal.signal(sig, lambda *_: setattr(guard, 'interrupted', True))
|
||||
for sig in (signal.SIGTERM, signal.SIGINT)}
|
||||
runtime = env = hooks = recorder = gui = None
|
||||
result = {'success': False, 'mode': payload['mode'], 'optimizer_steps': 0, 'accepted_skill': False,
|
||||
'environment_constructed': False, 'physics_counts': None, 'runtime_continuity_verified': False}
|
||||
expected = inventory = None
|
||||
code = 1
|
||||
try:
|
||||
guard.check()
|
||||
if payload['sources'] != (skill.sources() if is_skill else followup.sources() if is_followup else launch.sources()) or payload['launch_identity'] != launch.owner.launch_identity():
|
||||
raise ValueError('identity_worker_source_or_launch_drift')
|
||||
runtime = actual_runtime(payload)
|
||||
mod = runtime.modules
|
||||
if is_skill and payload['mode'] == 'run' and payload['spec']['gui']:
|
||||
def live_view_guard():
|
||||
guard.check()
|
||||
mod['live_cadence_v042'].require_recorder_live(recorder)
|
||||
gui = mod['skill_gui_v041'].LivePresentation(runtime.app, live_view_guard,
|
||||
lambda value: guard.write(output / 'gui-status.json', value))
|
||||
binder = runtime.binding_module
|
||||
hook_type = mod['evaluation_env'].RuntimeHooks
|
||||
if payload['mode'] == 'prepare':
|
||||
# Protect config preparation against any accidental task physics call.
|
||||
def no_physics(h):
|
||||
raise RuntimeError('budget_prepare_forbids_sim_step')
|
||||
hooks = hook_type.isaac(before_step=no_physics)
|
||||
hooks.__enter__() # lifetime ends only after guarded close in finally
|
||||
if hooks.active:
|
||||
inventory = binder.propose_inventory(ROOT, runtime.cfg, launch.owner.proposed_native_layouts)
|
||||
expected = binder.snapshot(ROOT, runtime.cfg, inventory)
|
||||
guard.write(output / 'inventory.json', inventory)
|
||||
guard.write(output / 'binding.json', expected)
|
||||
verify_runtime(runtime, expected, inventory)
|
||||
result.update(success=True, environment_constructed=False,
|
||||
physics_counts=mod['functional_flow_recording_v2'].counters(hooks),
|
||||
runtime_qualified=False, config_prepared=True,
|
||||
binding_sha256=b.digest(output / 'binding.json'),
|
||||
inventory_sha256=b.digest(output / 'inventory.json'))
|
||||
if is_skill:
|
||||
result['kit_runtime_metadata'] = mod['performance_v042'].runtime_metadata(runtime.app)
|
||||
else:
|
||||
expected = launch.read(payload['binding'], payload['binding_sha256'])
|
||||
inventory = launch.read(payload['inventory'], payload['inventory_sha256'])
|
||||
verify_runtime(runtime, expected, inventory)
|
||||
cfg = mod['functional_flow_v2'].load_contract(ROOT / launch.CONFIG)
|
||||
cycles, kind = payload['cycles'], payload['controller_kind']
|
||||
if is_skill:
|
||||
spec = skill.validate_spec(payload['spec'])
|
||||
if cycles != spec['cycles'] or kind != mod['skill_v4'].CONTROLLERS[spec['controller']]:
|
||||
raise ValueError('Authenticated v4 slot/spec mismatch')
|
||||
session = mod['skill_v4'].SkillSession(spec['command'], cycles, spec['controller'], spec['queued'])
|
||||
protocol = mod['skill_v4'].SkillProtocol(cfg, session)
|
||||
result['skill_spec'] = spec
|
||||
elif is_followup:
|
||||
if cycles != 3:
|
||||
raise ValueError('Followup requires exactly three continuous cycles')
|
||||
p = mod['functional_followup_v3'].load_plan(ROOT / followup.PLAN)
|
||||
protocol = mod['functional_followup_v3'].FollowupProtocol(cfg, p, payload['command_id'], kind)
|
||||
result['command_id'] = payload['command_id']
|
||||
else:
|
||||
protocol = mod['functional_flow_v2'].FunctionalFlowProtocol(cfg, cycles=cycles, controller_kind=kind)
|
||||
frozen, _ = mod['replay'].verify_bundle(ROOT)
|
||||
policy = (mod['replay'].load_frozen_policy(ROOT / mod['replay'].CANDIDATE, frozen).to('cuda:0')
|
||||
if kind == 'learned_policy' else None)
|
||||
# Install budget/count/reset/pose guards BEFORE environment construction.
|
||||
def before_physics(h):
|
||||
guard.check()
|
||||
if gui is not None and (gui.closed or not runtime.app.is_running()):
|
||||
raise RuntimeError('gui_window_closed_before_completion')
|
||||
if is_skill:
|
||||
mod['live_cadence_v042'].require_recorder_live(recorder)
|
||||
if (h.mode != 'rollout' and h.initialization_steps >= 250
|
||||
or h.mode == 'rollout' and h.rollout_steps >= cycles * 4500
|
||||
or h.completed_steps >= cycles * 4500 + 250):
|
||||
raise RuntimeError('budget_physics_call_cap')
|
||||
recorder = (mod['performance_v042'].recorder_class(ROOT)(
|
||||
cfg, protocol, guard.check, Path(payload['mailbox']),
|
||||
stop_at_tick=spec['stop_at_tick'], reject_at_tick=spec['reject_at_tick']) if is_skill else
|
||||
mod['functional_flow_recording_v2'].FunctionalFlowRecorder(cfg, protocol, guard.check))
|
||||
hooks = hook_type.isaac(before_step=before_physics)
|
||||
hooks.__enter__() # also guard exception-path close
|
||||
if hooks.active:
|
||||
result['environment_construction_entered'] = True
|
||||
guard.write(output / 'worker-progress.json', result)
|
||||
env = mod['evaluation_env'].EvaluationContactCycleEnv(
|
||||
cfg=runtime.cfg, protocol=protocol, trial_index=0, runtime_hooks=hooks,
|
||||
evidence_callback=recorder, seam_callback=recorder.seam_callback,
|
||||
**({'boundary_stop_callback': lambda: session.stop_requested} if is_skill else {}))
|
||||
result['environment_constructed'] = True
|
||||
s = mod['evaluation_mdp'].state(env)
|
||||
if (s.robot.is_fixed_base or runtime.cfg.sim.physics.solver_type != 0
|
||||
or runtime.cfg.sim.physics.enable_external_forces_every_iteration
|
||||
or runtime.cfg.terminations.failed is None or env.physics_dt != .002
|
||||
or env.step_dt != .01 or bool(s.scenarios.any() or s.stand_only.any())):
|
||||
raise RuntimeError('identity_require_frozen_floating_root_nominal_failure_enabled_fixture')
|
||||
result['fixture'] = {'floating_root': True, 'num_envs': env.num_envs,
|
||||
'system_weight_N': float(s.weight[0]), 'physics_dt_s': env.physics_dt,
|
||||
'control_dt_s': env.step_dt, 'guide_enabled': runtime.cfg.guide_enabled,
|
||||
'auxiliary_enabled': runtime.cfg.auxiliary_enabled}
|
||||
verify_runtime(runtime, expected, inventory) # constructor cannot silently mutate cfg/imports
|
||||
if is_skill:
|
||||
result['performance_installation'] = mod['performance_v042'].install_readonly_path(ROOT, env)
|
||||
recorder.attach(env)
|
||||
obs, _ = env.reset()
|
||||
if is_followup:
|
||||
pair = followup.initial_pair_record(recorder, s, obs['policy'], payload['command_id'])
|
||||
guard.write(output / 'initial-pair.json', pair)
|
||||
result['initial_pair_sha256'] = b.digest(output / 'initial-pair.json')
|
||||
result['initial_pair_matched'] = False
|
||||
if payload.get('pair_reference') is not None:
|
||||
expected_pair = launch.read(payload['pair_reference'], payload['pair_reference_sha256'])
|
||||
result['initial_pair_matched'] = followup.require_pair(pair, expected_pair)
|
||||
stop = env.stop_result
|
||||
if gui is not None:
|
||||
gui.start(env)
|
||||
if is_skill and mod['performance_v042'].options(ROOT)['mode'] == 'combined_display':
|
||||
mod['live_cadence_v042'].attach(recorder, env, gui)
|
||||
if gui.closed:
|
||||
stop = env._stop(('gui_window_closed_before_completion',))
|
||||
controls = 0
|
||||
profiler = mod['performance_v042'].LoopProfile(ROOT, env) if is_skill else None
|
||||
if profiler is not None:
|
||||
profiler.start()
|
||||
with runtime.torch.inference_mode():
|
||||
while stop is None:
|
||||
guard.check()
|
||||
x = obs['policy']
|
||||
action, mean, std = mod['replay'].inference_action(
|
||||
x, 'learned' if policy is not None else 'reference', policy)
|
||||
recorder.before_control(x, action, mean, std)
|
||||
stepped = env.step(action)
|
||||
if is_skill:
|
||||
# Includes the original interval20 render at the
|
||||
# final substep, even on the final protocol tick.
|
||||
mod['live_cadence_v042'].require_recorder_live(recorder)
|
||||
controls += 1
|
||||
stop = stepped.stop
|
||||
obs = stepped.observation
|
||||
if gui is not None:
|
||||
gui.after_control(env)
|
||||
if stop is None and (gui.closed or not runtime.app.is_running()):
|
||||
stop = env._stop(('gui_window_closed_before_completion',))
|
||||
if controls % 100 == 0 or stop is not None:
|
||||
guard.write(output / 'physics-counts.json', {
|
||||
**mod['functional_flow_recording_v2'].counters(hooks),
|
||||
'physics_samples': len(recorder.frames), 'controls_entered': controls,
|
||||
'stop': None if stop is None else asdict(stop)})
|
||||
if stepped.reward is not None and not bool(runtime.torch.isfinite(stepped.reward).all()):
|
||||
raise RuntimeError('nonfinite_reward')
|
||||
if profiler is not None:
|
||||
guard.write(output / 'performance-profile.json', profiler.finish(controls, len(recorder.frames)))
|
||||
continuity = recorder.continuity(stop, hooks, cycles)
|
||||
if is_skill:
|
||||
result.update(live_receipts=session.receipts, pair_commands=session.pair_commands,
|
||||
cycles_completed=stop.completed_clock_cycles)
|
||||
recorder.status(env._event('tick', stop.causes), s, state='stopped', stop=stop)
|
||||
result.update(controller_kind=kind, cycles_requested=cycles, control_calls=controls,
|
||||
runtime_stop=asdict(stop), cycle_results=recorder.cycles,
|
||||
continuity=continuity, runtime_continuity_verified=continuity['runtime_continuity_verified'],
|
||||
success=continuity['runtime_continuity_verified'],
|
||||
physics_counts=mod['functional_flow_recording_v2'].counters(hooks))
|
||||
if gui is not None:
|
||||
# Keep the physical scene alive, not a screenshot/trajectory
|
||||
# replay. No env.step after completion; close is intercepted
|
||||
# until evidence can be finalized by this same owner.
|
||||
if result['success']:
|
||||
recorder.status(env._event('tick', stop.causes), s, state='viewing_final', stop=stop)
|
||||
gui.hold(env, result['success'] and stop.reason == 'protocol_complete',
|
||||
safely_stopped=result['success'] and stop.reason == 'requested_stop')
|
||||
if is_skill:
|
||||
mod['live_cadence_v042'].require_recorder_live(recorder)
|
||||
result['gui'] = gui.report()
|
||||
# Close remains guarded; no reset/pose writes or extra physical steps.
|
||||
env.close()
|
||||
env = None
|
||||
verify_runtime(runtime, expected, inventory)
|
||||
result['preclose_identity_verified'] = True
|
||||
result['runtime'] = {'torch': str(runtime.torch.__version__), 'device': 'cuda:0',
|
||||
'gpu': runtime.torch.cuda.get_device_name(0),
|
||||
'rng_inventory': runtime.rng_inventory}
|
||||
code = 0 if result['success'] else 2
|
||||
except BaseException as error:
|
||||
result.update(success=False, error=traceback.format_exc())
|
||||
if hasattr(error, 'binding_diagnostic'):
|
||||
result['binding_diagnostic'] = error.binding_diagnostic
|
||||
traceback.print_exc()
|
||||
code = 1
|
||||
finally:
|
||||
if env is not None:
|
||||
try:
|
||||
env.close()
|
||||
except BaseException:
|
||||
result['close_error'] = traceback.format_exc()
|
||||
result['success'], code = False, 1
|
||||
if hooks is not None:
|
||||
hooks.__exit__(None, None, None)
|
||||
result['physics_counts'] = {k: getattr(hooks, k) for k in (
|
||||
'attempted_steps', 'completed_steps', 'initialization_steps', 'rollout_steps',
|
||||
'reset_calls', 'pose_write_calls', 'composer_buffer_replacements')}
|
||||
result['hook_violations'] = hooks.violations
|
||||
try:
|
||||
if is_skill:
|
||||
status_path = Path(payload['mailbox']) / 'status.json'
|
||||
status = skill.mailbox().read_json(status_path)
|
||||
if not result['success']:
|
||||
status.update(state='failed', failure=[result.get('error', 'runtime_or_integrity_failure')[-4096:]])
|
||||
status['worker_success'] = result['success']
|
||||
skill.mailbox().publish(status_path, status)
|
||||
if recorder is not None:
|
||||
guard.tensor(output / 'initial-state.pt', recorder.initial, cleanup=True)
|
||||
guard.tensor(output / 'trace.pt', recorder.trace(), cleanup=True)
|
||||
guard.write(output / 'events-and-seams.json', {'events': recorder.events,
|
||||
'seams': recorder.seams, 'pending_seam': recorder.pending_seam,
|
||||
'recording_faults': recorder.faults,
|
||||
**({'live_receipts': recorder.session.receipts, 'pair_commands': recorder.session.pair_commands}
|
||||
if is_skill else {})}, cleanup=True)
|
||||
if gui is not None:
|
||||
result['gui'] = gui.report()
|
||||
result['worker_final_boottime'] = b.boottime()
|
||||
if is_skill:
|
||||
result['shutdown_handoff_ready'] = True
|
||||
guard.write(output / 'worker-result.json', result, cleanup=True)
|
||||
if is_skill:
|
||||
# Keep this owned GPU process alive until the supervisor has
|
||||
# made its last strict LIVE ownership observation. Only then
|
||||
# may app.close/exit race with process discovery in CLEANUP.
|
||||
skill.wait_shutdown_release(output, guard)
|
||||
except BaseException:
|
||||
traceback.print_exc()
|
||||
code = 1
|
||||
for sig, handler in interrupted.items():
|
||||
signal.signal(sig, handler)
|
||||
if gui is not None:
|
||||
gui.release()
|
||||
if runtime is not None:
|
||||
runtime.app.close(exit_code=code)
|
||||
return code
|
||||
|
||||
|
||||
def release_shutdown(output, guard, gpu, owned):
|
||||
"""Explicit LIVE -> bounded CLEANUP transition, never an NVML tolerance.
|
||||
|
||||
Worker waits alive after all guarded physics/env close/result publication.
|
||||
The original strict live check must PASS before its exit is released. Strict
|
||||
post-wait final GPU check remains mandatory. No unknown client is waived.
|
||||
"""
|
||||
path = output / 'worker-result.json'
|
||||
if not path.exists():
|
||||
return None
|
||||
receipt = launch.read(path)
|
||||
if receipt.get('shutdown_handoff_ready') is not True:
|
||||
raise RuntimeError('process_missing_shutdown_handoff')
|
||||
observation = gpu.check(owned)
|
||||
digest = b.digest(path)
|
||||
guard.write(output / 'shutdown-release.json', {'worker_result_sha256': digest})
|
||||
return {'released': True, 'worker_result_sha256': digest, 'last_live_gpu': observation,
|
||||
'scope': 'No physics after receipt; bounded owned cleanup then unchanged strict final GPU check'}
|
||||
|
||||
|
||||
def filesystem_verify(binding):
|
||||
# Reuse finite binding file records, not a new recursive filesystem traversal.
|
||||
files = {}
|
||||
def visit(x):
|
||||
if isinstance(x, dict):
|
||||
if isinstance(x.get('path'), str) and isinstance(x.get('sha256'), str):
|
||||
files[x['path']] = x['sha256']
|
||||
for item in x.values():
|
||||
visit(item)
|
||||
elif isinstance(x, list):
|
||||
for item in x:
|
||||
visit(item)
|
||||
visit(binding)
|
||||
files.update(binding['runtime_layout_pins'])
|
||||
for filename, digest in files.items():
|
||||
if b.digest(Path(filename)) != digest:
|
||||
raise ValueError('identity_postexit_file_changed: ' + filename)
|
||||
return {'files_rehashed': len(files), 'scope': 'static finite files; preclose same-object cfg checked by worker'}
|
||||
|
||||
|
||||
def campaign(args):
|
||||
root = args.output
|
||||
is_followup = getattr(args, 'followup_v3', False)
|
||||
is_skill = getattr(args, 'skill_v4', False)
|
||||
if is_skill:
|
||||
import skill_launch_v4 as skill
|
||||
approved = skill.approval(args.approval, args.trusted_approval_sha256, root)
|
||||
elif is_followup:
|
||||
import functional_followup_launch_v3 as followup
|
||||
approved = followup.approval(args.approval, args.trusted_approval_sha256, root)
|
||||
else:
|
||||
approved = launch.approval(args.approval, args.trusted_approval_sha256, root)
|
||||
limits = approved['limits']
|
||||
slot_rows = (skill.slots(approved['spec']) if is_skill else followup.api().slots(followup.plan()) if is_followup else
|
||||
[{'slot': name, 'cycles': cycles, 'controller_kind': kind} for name, cycles, kind in launch.SLOTS])
|
||||
reserved_calls = limits['max_physics_calls']
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise RuntimeError('Subreaper unavailable')
|
||||
root.mkdir()
|
||||
start = b.boottime()
|
||||
deadline = min(start + limits['max_campaign_seconds'], approved['deadline_boottime']) if is_skill else start + limits['max_campaign_seconds']
|
||||
campaign_guard = launch.Guard(root, start, deadline, cleanup_reserve=0,
|
||||
cap=limits['max_owned_bytes'], stdout_slots=len(slot_rows))
|
||||
campaign_guard.write(root / 'approval.json', approved)
|
||||
if is_skill:
|
||||
mailbox_path = root / 'mailbox'
|
||||
mailbox_path.mkdir()
|
||||
skill.mailbox().publish(mailbox_path / 'status.json', {'state': 'starting', 'global_tick': 0,
|
||||
'failure': [], 'receipts_count': 0})
|
||||
campaign_guard.write(root / 'reservation.json', {
|
||||
'approved_by': 'current user 批准', 'start_boottime': start, 'deadline_boottime': deadline,
|
||||
'limits': limits, 'reserved_physics_calls': reserved_calls, 'actual_final_calls': None,
|
||||
'note': 'Reserved ceiling is not actual count; no retries/refunds/old grants.'})
|
||||
policy = launch.native_policy(root, deadline, cap=limits['max_owned_bytes'], slot_names=[s['slot'] for s in slot_rows])
|
||||
campaign_guard.write(root / 'native-policy.json', policy)
|
||||
monitor = launch.AsyncMonitor(policy)
|
||||
gpu = None
|
||||
results = []
|
||||
final = {'success': False, 'rows': results, 'optimizer_steps': 0, 'accepted_skill': False}
|
||||
handlers = {sig: signal.signal(sig, lambda *_: setattr(campaign_guard, 'interrupted', True))
|
||||
for sig in (signal.SIGTERM, signal.SIGINT)}
|
||||
binding = inventory = None
|
||||
try:
|
||||
# Wait for one finite baseline scan, without consuming a Kit launch.
|
||||
while monitor.monitor is None:
|
||||
campaign_guard.check(); monitor.check(); time.sleep(.05)
|
||||
if is_followup or is_skill:
|
||||
from functional_gpu_watch_v3 import ObservedGPUOwnership
|
||||
gpu = ObservedGPUOwnership(approved['gpu_identity'],
|
||||
lambda event: campaign_guard.write(root / 'gpu-observations.json', event, cleanup=True))
|
||||
else:
|
||||
gpu = launch.ownership.GPUOwnership(approved['gpu_identity'])
|
||||
for index, slot in enumerate(slot_rows):
|
||||
name, cycles, kind = slot['slot'], slot['cycles'], slot['controller_kind']
|
||||
campaign_guard.check(); monitor.check()
|
||||
if is_skill:
|
||||
args.skill_reservation.check_storage(args.skill_reservation.runs)
|
||||
if is_followup and index >= 2 and not results[1].get('success'):
|
||||
raise RuntimeError('New supervised u256 nominal three-cycle required before matrix')
|
||||
if not is_followup and not is_skill and index >= 3 and not all(x.get('success') for x in results[1:3]):
|
||||
raise RuntimeError('Both single cycles required before continuous runs')
|
||||
output = root / name
|
||||
output.mkdir()
|
||||
native = root / 'native' / name
|
||||
native.mkdir(parents=True)
|
||||
environment = os.environ.copy()
|
||||
for var in ('VIRTUAL_ENV', 'CONDA_PREFIX', 'PYTHONEXE', 'PYTHONHOME', 'args'):
|
||||
environment.pop(var, None)
|
||||
redirects = launch.owner.native_redirects(native, preparation=True)
|
||||
for path in redirects.values():
|
||||
Path(path).mkdir()
|
||||
environment.update(redirects)
|
||||
environment.update(PYTHONPATH=str(ROOT / 'source/SkillFactory') + ':' + str(ROOT / 'scripts'),
|
||||
PYTHONUNBUFFERED='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
|
||||
CUDA_VISIBLE_DEVICES='0', HEADLESS='0' if is_skill and approved['spec']['gui'] and index else '1',
|
||||
LIVESTREAM='0')
|
||||
if is_skill:
|
||||
# Frozen import-time asset defaults predate relocatable packaging.
|
||||
# Select this tree without changing their immutable source bytes.
|
||||
environment.update(SKILLFACTORY_GO2_ASSET_DIR=str(ROOT / 'assets/generated/go2_fl'),
|
||||
SKILLFACTORY_LEG_BUNDLE=str(ROOT / 'assets/generated/legs/go2_fl'))
|
||||
slot_start = b.boottime()
|
||||
seconds = (limits['max_prepare_seconds' if index == 0 else 'max_physical_seconds']
|
||||
if is_followup or is_skill else limits['max_launch_seconds'])
|
||||
slot_deadline = min(slot_start + seconds, deadline)
|
||||
guard = launch.Guard(root, slot_start, slot_deadline, cap=limits['max_owned_bytes'], stdout_slots=len(slot_rows))
|
||||
guard.check()
|
||||
payload = {'mode': 'prepare' if index == 0 else 'run', 'slot': name, 'root': str(root),
|
||||
'output': str(output), 'native': str(native), 'start': slot_start,
|
||||
'deadline': slot_deadline, 'cycles': cycles, 'controller_kind': kind,
|
||||
'sources': approved['sources'], 'launch_identity': launch.owner.launch_identity(),
|
||||
'followup_v3': is_followup, 'owned_cap': limits['max_owned_bytes']}
|
||||
if is_skill:
|
||||
payload.update(skill_v4=True, spec=approved['spec'], mailbox=str(mailbox_path))
|
||||
if is_followup:
|
||||
payload.update(command_id=slot['command_id'], pair_reference=None)
|
||||
if slot['pair_group'] is not None and kind == 'learned_policy':
|
||||
reference = results[-1]
|
||||
if not reference['success'] or reference['slot'] != slot['pair_group'] + '-reference':
|
||||
raise RuntimeError('Complete matching reference required before paired u256')
|
||||
path = root / reference['slot'] / 'initial-pair.json'
|
||||
payload.update(pair_reference=str(path), pair_reference_sha256=b.digest(path))
|
||||
if index:
|
||||
payload.update(binding=str(root / 'prepare/binding.json'), binding_sha256=b.digest(root / 'prepare/binding.json'),
|
||||
inventory=str(root / 'prepare/inventory.json'), inventory_sha256=b.digest(root / 'prepare/inventory.json'))
|
||||
owned = launch.owner.OwnedProcesses()
|
||||
gpu.check(owned, final=True)
|
||||
parent, child_channel = socket.socketpair()
|
||||
command = launch.owner.launch_command([str(ROOT / (skill.SCRIPT if is_skill else launch.SCRIPT)), '--worker-fd', str(child_channel.fileno())], environment)
|
||||
# Persist the consumed slot BEFORE any child/AppLauncher starts.
|
||||
campaign_guard.write(output / 'reservation.json', {'index': index, 'payload': payload,
|
||||
'command': command, 'physics_ceiling': 0 if index == 0 else cycles * 4500 + 250})
|
||||
child = capture = None
|
||||
shutdown_handoff = None
|
||||
clean = False
|
||||
row = {'slot': name, 'success': False, 'status': 'reserved', 'pair_group': slot.get('pair_group')}
|
||||
results.append(row)
|
||||
try:
|
||||
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, start_new_session=True,
|
||||
pass_fds=(child_channel.fileno(),))
|
||||
child_channel.close()
|
||||
owned.attach(child)
|
||||
capture = launch.owner.StdoutCapture(child.stdout, output / 'stdout.log', b)
|
||||
launch.authenticate(parent, child, payload, capture, guard, monitor, owned, gpu)
|
||||
parent.close()
|
||||
while child.poll() is None:
|
||||
campaign_guard.check(); guard.check(); monitor.check(); capture.check()
|
||||
if is_skill and shutdown_handoff is None:
|
||||
shutdown_handoff = release_shutdown(output, guard, gpu, owned)
|
||||
if shutdown_handoff is None:
|
||||
gpu.check(owned)
|
||||
if is_skill and b.boottime() >= getattr(args, '_next_storage_check', 0):
|
||||
args._next_storage_check = b.boottime() + 1
|
||||
args.skill_reservation.check_storage(args.skill_reservation.runs)
|
||||
time.sleep(.1)
|
||||
capture.check()
|
||||
except BaseException:
|
||||
row['supervisor_error'] = traceback.format_exc()
|
||||
finally:
|
||||
parent.close(); child_channel.close()
|
||||
if child is not None:
|
||||
clean = launch.owner.cleanup_group(child, slot_deadline, b, owned,
|
||||
capture.pump if capture is not None else lambda: None)
|
||||
child.wait(timeout=max(.01, slot_deadline - b.boottime()))
|
||||
row.update(returncode=child.returncode, cleanup_clean=clean, waited_exit=True,
|
||||
owned_identities=sorted([list(x) for x in owned.known]))
|
||||
if capture is not None:
|
||||
capture.close()
|
||||
try:
|
||||
row['gpu_postexit'] = gpu.check(owned, final=True)
|
||||
except BaseException:
|
||||
row['gpu_postexit_error'] = traceback.format_exc()
|
||||
row.setdefault('supervisor_error', row['gpu_postexit_error'])
|
||||
row['end_boottime'] = b.boottime()
|
||||
worker_path = output / 'worker-result.json'
|
||||
if worker_path.exists():
|
||||
receipt = launch.read(worker_path)
|
||||
row['worker_result_sha256'] = b.digest(worker_path)
|
||||
row['worker'] = receipt
|
||||
row['success'] = bool(receipt.get('success') and receipt.get('preclose_identity_verified')
|
||||
and row.get('returncode') == 0 and clean and not row.get('supervisor_error'))
|
||||
if is_skill and index and not skill.complete_receipt(receipt, approved['spec']):
|
||||
row['success'] = False
|
||||
row['incomplete_worker_receipt'] = True
|
||||
if is_followup and not followup.complete_receipt(receipt, slot):
|
||||
row['success'] = False
|
||||
row['incomplete_worker_receipt'] = True
|
||||
if is_skill:
|
||||
row['shutdown_handoff'] = shutdown_handoff
|
||||
if (shutdown_handoff is None or not shutdown_handoff['released']
|
||||
or shutdown_handoff['worker_result_sha256'] != row.get('worker_result_sha256')):
|
||||
row['success'] = False
|
||||
row['shutdown_handoff_missing_or_changed'] = True
|
||||
if index == 0 and row['success']:
|
||||
binding = launch.read(root / 'prepare/binding.json')
|
||||
inventory = launch.read(root / 'prepare/inventory.json')
|
||||
expected_format = ('skillfactory.go2-fl-skill-binding.v4' if is_skill else
|
||||
'skillfactory.functional-followup-binding.v3' if is_followup
|
||||
else 'skillfactory.contact-cycle-functional-binding.v2')
|
||||
if (binding.get('format') != expected_format
|
||||
or is_followup and binding.get('plan_sha256') != approved['sources'][followup.PLAN]
|
||||
or binding.get('contract_sha256') != approved['sources'][launch.CONFIG]
|
||||
or row['worker']['environment_constructed'] is not False
|
||||
or row['worker']['physics_counts']['completed_steps'] != 0):
|
||||
raise RuntimeError('Config-only parent review failed')
|
||||
row['parent_review'] = filesystem_verify(binding)
|
||||
# Preserve the exact prepared sources, not authority copied from old runs.
|
||||
for relative in approved['sources']:
|
||||
dest = root / 'source_snapshot' / relative
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
b.durable_write(dest, (ROOT / relative).read_bytes())
|
||||
campaign_guard.write(root / 'preparation-reviewed.json', {
|
||||
'binding_sha256': b.digest(root / 'prepare/binding.json'),
|
||||
'inventory_sha256': b.digest(root / 'prepare/inventory.json'),
|
||||
'approved_from_current_user_budget_only': True, 'config_only_not_physical_qualification': True,
|
||||
'static_verification': row['parent_review']})
|
||||
elif index and row['success']:
|
||||
row['postexit_files'] = filesystem_verify(binding)
|
||||
if is_followup and row['success'] and slot['pair_group'] is not None and kind == 'learned_policy':
|
||||
row['paired_comparison'] = followup.paired_comparison(results[-2], row)
|
||||
if not row['paired_comparison']['eligible']:
|
||||
row['success'] = False
|
||||
if (is_followup or is_skill) and index:
|
||||
row['artifact_sha256'] = {}
|
||||
for filename in (('trace.pt', 'initial-state.pt', 'events-and-seams.json') if is_skill else
|
||||
('trace.pt', 'initial-state.pt', 'events-and-seams.json', 'initial-pair.json')):
|
||||
path = output / filename
|
||||
if not path.is_file():
|
||||
row['success'] = False
|
||||
row.setdefault('missing_required_results', []).append(filename)
|
||||
else:
|
||||
row['artifact_sha256'][filename] = b.digest(path)
|
||||
if is_followup and row['artifact_sha256'].get('initial-pair.json') != row.get('worker', {}).get('initial_pair_sha256'):
|
||||
row['success'] = False
|
||||
row['initial_pair_file_identity_mismatch'] = True
|
||||
campaign_guard.write(output / 'supervisor-result.json', row)
|
||||
campaign_guard.write(root / 'campaign-progress.json', final)
|
||||
if not row['success']:
|
||||
break # first failure consumes this campaign; never retry or tune
|
||||
monitor.check()
|
||||
final['success'] = len(results) == len(slot_rows) and all(x['success'] for x in results)
|
||||
except BaseException:
|
||||
final['error'] = traceback.format_exc()
|
||||
final['success'] = False
|
||||
if results:
|
||||
results[-1]['success'] = False
|
||||
results[-1].setdefault('supervisor_error', final['error'])
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
monitor.stop.set()
|
||||
if monitor.error is not None:
|
||||
final['success'] = False
|
||||
final['native_monitor'] = monitor.report()
|
||||
final['end_boottime'] = b.boottime()
|
||||
counts = [r.get('worker', {}).get('physics_counts') for r in results]
|
||||
final['actual_final_physics_calls'] = (sum(x['completed_steps'] for x in counts)
|
||||
if all(x is not None for x in counts) else None)
|
||||
final['reserved_physics_calls'] = reserved_calls
|
||||
if final['actual_final_physics_calls'] is None or final['actual_final_physics_calls'] > reserved_calls:
|
||||
final['success'] = False
|
||||
final['remaining_slots'] = [s['slot'] for s in slot_rows[len(results):]]
|
||||
try:
|
||||
campaign_guard.write(root / 'campaign-result.json', final, cleanup=True)
|
||||
if is_skill:
|
||||
status = skill.mailbox().read_json(mailbox_path / 'status.json')
|
||||
status.update(state='stopped' if final['success'] else 'failed',
|
||||
supervisor_success=final['success'], failure=status.get('failure', []) if final['success']
|
||||
else status.get('failure') or [final.get('error', 'supervision_or_worker_failure')[-4096:]])
|
||||
skill.mailbox().publish(mailbox_path / 'status.json', status)
|
||||
finally:
|
||||
for sig, handler in handlers.items():
|
||||
signal.signal(sig, handler)
|
||||
return 0 if final['success'] else 2
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--campaign', action='store_true')
|
||||
mode.add_argument('--worker-fd', type=int)
|
||||
mode.add_argument('--check-followup-v3', action='store_true', help='CPU metadata only; no grant, Kit or physics')
|
||||
parser.add_argument('--followup-v3', action='store_true', help='fresh u2563 then six paired command groups; new grant only')
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--approval', type=Path)
|
||||
parser.add_argument('--trusted-approval-sha256')
|
||||
args = parser.parse_args()
|
||||
if args.check_followup_v3:
|
||||
import functional_followup_launch_v3 as followup
|
||||
p = followup.plan()
|
||||
print(json.dumps({'execution_authorized': False, 'new_physics_calls': 0, 'optimizer_steps': 0,
|
||||
'plan_sha256': b.digest(ROOT / followup.PLAN), 'plan': p,
|
||||
'commands_m': followup.api().commands(p), 'slots': followup.api().slots(p),
|
||||
'sources': followup.sources()}, indent=2))
|
||||
return 0
|
||||
if args.worker_fd is not None:
|
||||
return worker(launch.worker_payload(args.worker_fd))
|
||||
if args.output is None or args.approval is None or args.trusted_approval_sha256 is None:
|
||||
parser.error('Require fresh independently trusted approval and new output root')
|
||||
return campaign(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user