Files
SkillFactory/scripts/evaluate_cycle_functional_flow.py
chenlin c528dd268f feat: checkpoint Go2 FL simulation performance v0.4.2
Include callable skill and live-GUI prerequisites, epoch sensor reads, packed snapshots, sampled bit-exact kinematics, and verified-tick rendering without changing the physical/control schedule.

Record bounded analytic S1 GUI/headless qualification, original failures and CPU evidence (1557 passed, 6 skipped). Preserve accepted_skill=false and sub-real-time limitations. Exclude native authorities, caches, large media and local delivery copies; retain sealed artifact digest.
2026-10-16 15:46:01 +08:00

724 lines
44 KiB
Python

#!/usr/bin/env python3
"""Separately approved functional campaigns; optional frozen --followup-v3.
Default: PREP + reference/u256 1/3 cycles. Followup: PREP + u2563 + six paired
three-cycle groups. No training/retry or old authority. --campaign is the sole
supervisor; workers require its inherited authenticated FD before AppLauncher.
"""
import argparse
import ctypes
import hashlib
import importlib
import json
import os
import signal
import socket
import subprocess
import sys
import time
import traceback
from dataclasses import asdict
from pathlib import Path
from types import SimpleNamespace
import functional_flow_launch_v2 as launch
ROOT, b = launch.ROOT, launch.b
def actual_runtime(payload):
"""Config-only setup shared by PREP and RUN. Never constructs an environment."""
from isaaclab.app import AppLauncher
native = Path(payload['native'])
if payload.get('skill_v4'):
import skill_launch_v4
gui_enabled = skill_launch_v4.gui_enabled(payload)
else:
gui_enabled = False
gui_args = (' --/app/window/title=Go2_FL_Contact_Cycle_0.4.1 --/app/window/width=1440'
' --/app/window/height=960 --/app/window/x=120 --/app/window/y=80'
' --/app/file/ignoreUnsavedOnExit=true') if gui_enabled else ''
app = AppLauncher(headless=not gui_enabled, visualizer='kit' if gui_enabled else 'none',
device='cuda:0', fast_shutdown=True,
kit_args=f'--/log/file={native / "kit.log"}' + gui_args).app
try:
# Source inventory learned in PREP is independently rehashed before RUN.
inventory = None if payload['mode'] == 'prepare' else launch.read(payload['inventory'], payload['inventory_sha256'])
for name in (() if inventory is None else inventory['modules']):
importlib.import_module(name)
task = 'SkillFactory.tasks.manager_based.contact_cycle_sim.'
leg = 'SkillFactory.legs.contact_cycle_sim.'
modules = {name: importlib.import_module(task + name) for name in (
'evaluation_cfg', 'evaluation_env', 'evaluation_mdp', 'evaluation_rng_scope',
'evaluation_instrumentation', 'functional_flow_recording_v2')}
modules.update({name: importlib.import_module(leg + name) for name in (
'functional_flow_v2', 'functional_flow_binding_v2', 'evaluation_binding', 'replay')})
if payload.get('followup_v3'):
for name in ('functional_followup_v3', 'functional_followup_binding_v3'):
modules[name] = importlib.import_module(leg + name)
importlib.import_module('functional_gpu_watch_v3')
if payload.get('skill_v4'):
for name in ('skill_v4', 'skill_mailbox_v4', 'skill_binding_v4',
'functional_followup_v3', 'functional_followup_binding_v3'):
modules[name] = importlib.import_module(leg + name)
modules['skill_recording_v4'] = importlib.import_module(task + 'skill_recording_v4')
modules['performance_v042'] = importlib.import_module(task + 'performance_v042')
modules['sensor_epoch_v042'] = importlib.import_module(task + 'sensor_epoch_v042')
modules['packed_recording_v042'] = importlib.import_module(task + 'packed_recording_v042')
modules['readonly_scope_v042'] = importlib.import_module(task + 'readonly_scope_v042')
modules['batched_kinematics_v042'] = importlib.import_module(task + 'batched_kinematics_v042')
modules['live_cadence_v042'] = importlib.import_module(task + 'live_cadence_v042')
if payload['spec']['gui']:
# PREP has no GUI experience, but must preload the exact finite
# presentation dependency before inventory/binding (no viewport
# construction or physics). RUN already enables this via Kit.
import omni.kit.app
manager = omni.kit.app.get_app().get_extension_manager()
for extension in ('omni.ui', 'omni.kit.viewport.utility', 'omni.replicator.core'):
if not manager.set_extension_enabled_immediate(extension, True):
raise RuntimeError('gui_extension_unavailable:' + extension)
modules['skill_gui_v041'] = importlib.import_module(task + 'skill_gui_v041')
if payload['spec']['gui']:
modules['skill_gui_v041'].prepare_native()
# GUI experience preloads Replicator. Match that exact module
# availability in headless PREP rather than relaxing its guard.
importlib.import_module(launch.owner.OPTIONAL_REPLICATOR)
for name in ('skill_launch_v4', 'functional_followup_launch_v3', 'functional_gpu_watch_v3',
'evaluate_cycle_functional_flow'):
importlib.import_module(name)
binding_module = modules['skill_binding_v4' if payload.get('skill_v4') else
'functional_followup_binding_v3' if payload.get('followup_v3') else 'functional_flow_binding_v2']
binder = modules['evaluation_binding']
for name in binder.REQUIRED_RUNTIME:
importlib.import_module(name)
instrument = modules['evaluation_instrumentation']
for name in instrument.BACKEND_SOURCE_INVENTORY:
importlib.import_module(name)
instrument.verify_backend_sources()
replicator = launch.owner.preimport_replicator()
if inventory is not None:
launch.owner.check_replicator_inventory(inventory)
import torch
torch.cuda.init()
torch.cuda.get_rng_state(torch.device('cuda:0'))
if torch.cuda.get_device_name(0) != 'NVIDIA GeForce RTX 5080':
raise ValueError('Require approved RTX5080')
cfg = modules['evaluation_cfg'].EvaluationContactCycleEnvCfg()
cfg.sim.device, cfg.seed = 'cuda:0', 93501
if payload.get('skill_v4') and payload['spec']['gui']:
# Same explicit presentation config/imports in PREP and RUN. Only
# RUN creates KitVisualizer/window; no physics config is changed.
from isaaclab_visualizers.kit import KitVisualizerCfg
importlib.import_module('isaaclab_visualizers.kit.kit_visualizer')
# Observed GUI constructor-only camera helper imports must be in
# PREP's exact provenance before construction, not waived afterward.
importlib.import_module('isaaclab_physx.renderers.kit_viewport_utils')
# Full GUI run 10: first render lazily loads this exact utility.
importlib.import_module('isaaclab_physx.renderers.isaac_rtx_renderer_utils')
cfg.sim.visualizer_cfgs = [KitVisualizerCfg()]
if payload.get('skill_v4'):
perf = modules['performance_v042']
perf.RUNTIME_MATH_QUALIFICATION = None
if perf.options(ROOT)['mode'] in ('combined', 'combined_display'):
nominal = torch.tensor([j['q_default'] for j in cfg.leg_bundle['spec']['joints']], device='cuda:0')
perf.RUNTIME_MATH_QUALIFICATION = modules['batched_kinematics_v042'].qualify(
cfg.leg_bundle, cfg.residual_parameters['reference'], nominal, 'cuda:0')
launch.owner.preforward_video_recorder(cfg, modules['evaluation_env'])
constructor_imports = launch.owner.preimport_constructor_modules(binder)
constructor_export, export_metadata = launch.owner.physx_constructor_export(binder, resolve=True)
config_imports = launch.owner.preimport_config_references(cfg, binder)
binder.normalize_config(cfg)
rng_scope = modules['evaluation_rng_scope']
rng_inventory = rng_scope.qualify_config(cfg)
rng_inventory.update(optional_replicator=replicator, constructor_imports=constructor_imports,
constructor_export=export_metadata, config_reference_imports=config_imports)
instrument.qualify_backend_rng(rng_scope.backend_rng(None))
return SimpleNamespace(app=app, modules=modules, cfg=cfg, torch=torch, binding_module=binding_module,
constructor_export=constructor_export, rng_inventory=rng_inventory)
except BaseException:
traceback.print_exc()
app.close(exit_code=1)
raise
def verify_runtime(runtime, expected, inventory):
binder = runtime.modules['evaluation_binding']
launch.owner.physx_constructor_export(binder, expected=runtime.constructor_export)
launch.owner.check_replicator_inventory(inventory)
return runtime.binding_module.verify(expected, ROOT, runtime.cfg, inventory)
def worker(payload):
root, output = Path(payload['root']), Path(payload['output'])
is_followup = payload.get('followup_v3', False)
is_skill = payload.get('skill_v4', False)
if is_skill:
import skill_launch_v4 as skill
if is_followup:
import functional_followup_launch_v3 as followup
guard = launch.Guard(root, payload['start'], payload['deadline'],
cap=payload.get('owned_cap', launch.CAP), stdout_slots=14 if is_followup else 5)
interrupted = {sig: signal.signal(sig, lambda *_: setattr(guard, 'interrupted', True))
for sig in (signal.SIGTERM, signal.SIGINT)}
runtime = env = hooks = recorder = gui = None
result = {'success': False, 'mode': payload['mode'], 'optimizer_steps': 0, 'accepted_skill': False,
'environment_constructed': False, 'physics_counts': None, 'runtime_continuity_verified': False}
expected = inventory = None
code = 1
try:
guard.check()
if payload['sources'] != (skill.sources() if is_skill else followup.sources() if is_followup else launch.sources()) or payload['launch_identity'] != launch.owner.launch_identity():
raise ValueError('identity_worker_source_or_launch_drift')
runtime = actual_runtime(payload)
mod = runtime.modules
if is_skill and payload['mode'] == 'run' and payload['spec']['gui']:
def live_view_guard():
guard.check()
mod['live_cadence_v042'].require_recorder_live(recorder)
gui = mod['skill_gui_v041'].LivePresentation(runtime.app, live_view_guard,
lambda value: guard.write(output / 'gui-status.json', value))
binder = runtime.binding_module
hook_type = mod['evaluation_env'].RuntimeHooks
if payload['mode'] == 'prepare':
# Protect config preparation against any accidental task physics call.
def no_physics(h):
raise RuntimeError('budget_prepare_forbids_sim_step')
hooks = hook_type.isaac(before_step=no_physics)
hooks.__enter__() # lifetime ends only after guarded close in finally
if hooks.active:
inventory = binder.propose_inventory(ROOT, runtime.cfg, launch.owner.proposed_native_layouts)
expected = binder.snapshot(ROOT, runtime.cfg, inventory)
guard.write(output / 'inventory.json', inventory)
guard.write(output / 'binding.json', expected)
verify_runtime(runtime, expected, inventory)
result.update(success=True, environment_constructed=False,
physics_counts=mod['functional_flow_recording_v2'].counters(hooks),
runtime_qualified=False, config_prepared=True,
binding_sha256=b.digest(output / 'binding.json'),
inventory_sha256=b.digest(output / 'inventory.json'))
if is_skill:
result['kit_runtime_metadata'] = mod['performance_v042'].runtime_metadata(runtime.app)
else:
expected = launch.read(payload['binding'], payload['binding_sha256'])
inventory = launch.read(payload['inventory'], payload['inventory_sha256'])
verify_runtime(runtime, expected, inventory)
cfg = mod['functional_flow_v2'].load_contract(ROOT / launch.CONFIG)
cycles, kind = payload['cycles'], payload['controller_kind']
if is_skill:
spec = skill.validate_spec(payload['spec'])
if cycles != spec['cycles'] or kind != mod['skill_v4'].CONTROLLERS[spec['controller']]:
raise ValueError('Authenticated v4 slot/spec mismatch')
session = mod['skill_v4'].SkillSession(spec['command'], cycles, spec['controller'], spec['queued'])
protocol = mod['skill_v4'].SkillProtocol(cfg, session)
result['skill_spec'] = spec
elif is_followup:
if cycles != 3:
raise ValueError('Followup requires exactly three continuous cycles')
p = mod['functional_followup_v3'].load_plan(ROOT / followup.PLAN)
protocol = mod['functional_followup_v3'].FollowupProtocol(cfg, p, payload['command_id'], kind)
result['command_id'] = payload['command_id']
else:
protocol = mod['functional_flow_v2'].FunctionalFlowProtocol(cfg, cycles=cycles, controller_kind=kind)
frozen, _ = mod['replay'].verify_bundle(ROOT)
policy = (mod['replay'].load_frozen_policy(ROOT / mod['replay'].CANDIDATE, frozen).to('cuda:0')
if kind == 'learned_policy' else None)
# Install budget/count/reset/pose guards BEFORE environment construction.
def before_physics(h):
guard.check()
if gui is not None and (gui.closed or not runtime.app.is_running()):
raise RuntimeError('gui_window_closed_before_completion')
if is_skill:
mod['live_cadence_v042'].require_recorder_live(recorder)
if (h.mode != 'rollout' and h.initialization_steps >= 250
or h.mode == 'rollout' and h.rollout_steps >= cycles * 4500
or h.completed_steps >= cycles * 4500 + 250):
raise RuntimeError('budget_physics_call_cap')
recorder = (mod['performance_v042'].recorder_class(ROOT)(
cfg, protocol, guard.check, Path(payload['mailbox']),
stop_at_tick=spec['stop_at_tick'], reject_at_tick=spec['reject_at_tick']) if is_skill else
mod['functional_flow_recording_v2'].FunctionalFlowRecorder(cfg, protocol, guard.check))
hooks = hook_type.isaac(before_step=before_physics)
hooks.__enter__() # also guard exception-path close
if hooks.active:
result['environment_construction_entered'] = True
guard.write(output / 'worker-progress.json', result)
env = mod['evaluation_env'].EvaluationContactCycleEnv(
cfg=runtime.cfg, protocol=protocol, trial_index=0, runtime_hooks=hooks,
evidence_callback=recorder, seam_callback=recorder.seam_callback,
**({'boundary_stop_callback': lambda: session.stop_requested} if is_skill else {}))
result['environment_constructed'] = True
s = mod['evaluation_mdp'].state(env)
if (s.robot.is_fixed_base or runtime.cfg.sim.physics.solver_type != 0
or runtime.cfg.sim.physics.enable_external_forces_every_iteration
or runtime.cfg.terminations.failed is None or env.physics_dt != .002
or env.step_dt != .01 or bool(s.scenarios.any() or s.stand_only.any())):
raise RuntimeError('identity_require_frozen_floating_root_nominal_failure_enabled_fixture')
result['fixture'] = {'floating_root': True, 'num_envs': env.num_envs,
'system_weight_N': float(s.weight[0]), 'physics_dt_s': env.physics_dt,
'control_dt_s': env.step_dt, 'guide_enabled': runtime.cfg.guide_enabled,
'auxiliary_enabled': runtime.cfg.auxiliary_enabled}
verify_runtime(runtime, expected, inventory) # constructor cannot silently mutate cfg/imports
if is_skill:
result['performance_installation'] = mod['performance_v042'].install_readonly_path(ROOT, env)
recorder.attach(env)
obs, _ = env.reset()
if is_followup:
pair = followup.initial_pair_record(recorder, s, obs['policy'], payload['command_id'])
guard.write(output / 'initial-pair.json', pair)
result['initial_pair_sha256'] = b.digest(output / 'initial-pair.json')
result['initial_pair_matched'] = False
if payload.get('pair_reference') is not None:
expected_pair = launch.read(payload['pair_reference'], payload['pair_reference_sha256'])
result['initial_pair_matched'] = followup.require_pair(pair, expected_pair)
stop = env.stop_result
if gui is not None:
gui.start(env)
if is_skill and mod['performance_v042'].options(ROOT)['mode'] == 'combined_display':
mod['live_cadence_v042'].attach(recorder, env, gui)
if gui.closed:
stop = env._stop(('gui_window_closed_before_completion',))
controls = 0
profiler = mod['performance_v042'].LoopProfile(ROOT, env) if is_skill else None
if profiler is not None:
profiler.start()
with runtime.torch.inference_mode():
while stop is None:
guard.check()
x = obs['policy']
action, mean, std = mod['replay'].inference_action(
x, 'learned' if policy is not None else 'reference', policy)
recorder.before_control(x, action, mean, std)
stepped = env.step(action)
if is_skill:
# Includes the original interval20 render at the
# final substep, even on the final protocol tick.
mod['live_cadence_v042'].require_recorder_live(recorder)
controls += 1
stop = stepped.stop
obs = stepped.observation
if gui is not None:
gui.after_control(env)
if stop is None and (gui.closed or not runtime.app.is_running()):
stop = env._stop(('gui_window_closed_before_completion',))
if controls % 100 == 0 or stop is not None:
guard.write(output / 'physics-counts.json', {
**mod['functional_flow_recording_v2'].counters(hooks),
'physics_samples': len(recorder.frames), 'controls_entered': controls,
'stop': None if stop is None else asdict(stop)})
if stepped.reward is not None and not bool(runtime.torch.isfinite(stepped.reward).all()):
raise RuntimeError('nonfinite_reward')
if profiler is not None:
guard.write(output / 'performance-profile.json', profiler.finish(controls, len(recorder.frames)))
continuity = recorder.continuity(stop, hooks, cycles)
if is_skill:
result.update(live_receipts=session.receipts, pair_commands=session.pair_commands,
cycles_completed=stop.completed_clock_cycles)
recorder.status(env._event('tick', stop.causes), s, state='stopped', stop=stop)
result.update(controller_kind=kind, cycles_requested=cycles, control_calls=controls,
runtime_stop=asdict(stop), cycle_results=recorder.cycles,
continuity=continuity, runtime_continuity_verified=continuity['runtime_continuity_verified'],
success=continuity['runtime_continuity_verified'],
physics_counts=mod['functional_flow_recording_v2'].counters(hooks))
if gui is not None:
# Keep the physical scene alive, not a screenshot/trajectory
# replay. No env.step after completion; close is intercepted
# until evidence can be finalized by this same owner.
if result['success']:
recorder.status(env._event('tick', stop.causes), s, state='viewing_final', stop=stop)
gui.hold(env, result['success'] and stop.reason == 'protocol_complete',
safely_stopped=result['success'] and stop.reason == 'requested_stop')
if is_skill:
mod['live_cadence_v042'].require_recorder_live(recorder)
result['gui'] = gui.report()
# Close remains guarded; no reset/pose writes or extra physical steps.
env.close()
env = None
verify_runtime(runtime, expected, inventory)
result['preclose_identity_verified'] = True
result['runtime'] = {'torch': str(runtime.torch.__version__), 'device': 'cuda:0',
'gpu': runtime.torch.cuda.get_device_name(0),
'rng_inventory': runtime.rng_inventory}
code = 0 if result['success'] else 2
except BaseException as error:
result.update(success=False, error=traceback.format_exc())
if hasattr(error, 'binding_diagnostic'):
result['binding_diagnostic'] = error.binding_diagnostic
traceback.print_exc()
code = 1
finally:
if env is not None:
try:
env.close()
except BaseException:
result['close_error'] = traceback.format_exc()
result['success'], code = False, 1
if hooks is not None:
hooks.__exit__(None, None, None)
result['physics_counts'] = {k: getattr(hooks, k) for k in (
'attempted_steps', 'completed_steps', 'initialization_steps', 'rollout_steps',
'reset_calls', 'pose_write_calls', 'composer_buffer_replacements')}
result['hook_violations'] = hooks.violations
try:
if is_skill:
status_path = Path(payload['mailbox']) / 'status.json'
status = skill.mailbox().read_json(status_path)
if not result['success']:
status.update(state='failed', failure=[result.get('error', 'runtime_or_integrity_failure')[-4096:]])
status['worker_success'] = result['success']
skill.mailbox().publish(status_path, status)
if recorder is not None:
guard.tensor(output / 'initial-state.pt', recorder.initial, cleanup=True)
guard.tensor(output / 'trace.pt', recorder.trace(), cleanup=True)
guard.write(output / 'events-and-seams.json', {'events': recorder.events,
'seams': recorder.seams, 'pending_seam': recorder.pending_seam,
'recording_faults': recorder.faults,
**({'live_receipts': recorder.session.receipts, 'pair_commands': recorder.session.pair_commands}
if is_skill else {})}, cleanup=True)
if gui is not None:
result['gui'] = gui.report()
result['worker_final_boottime'] = b.boottime()
if is_skill:
result['shutdown_handoff_ready'] = True
guard.write(output / 'worker-result.json', result, cleanup=True)
if is_skill:
# Keep this owned GPU process alive until the supervisor has
# made its last strict LIVE ownership observation. Only then
# may app.close/exit race with process discovery in CLEANUP.
skill.wait_shutdown_release(output, guard)
except BaseException:
traceback.print_exc()
code = 1
for sig, handler in interrupted.items():
signal.signal(sig, handler)
if gui is not None:
gui.release()
if runtime is not None:
runtime.app.close(exit_code=code)
return code
def release_shutdown(output, guard, gpu, owned):
"""Explicit LIVE -> bounded CLEANUP transition, never an NVML tolerance.
Worker waits alive after all guarded physics/env close/result publication.
The original strict live check must PASS before its exit is released. Strict
post-wait final GPU check remains mandatory. No unknown client is waived.
"""
path = output / 'worker-result.json'
if not path.exists():
return None
receipt = launch.read(path)
if receipt.get('shutdown_handoff_ready') is not True:
raise RuntimeError('process_missing_shutdown_handoff')
observation = gpu.check(owned)
digest = b.digest(path)
guard.write(output / 'shutdown-release.json', {'worker_result_sha256': digest})
return {'released': True, 'worker_result_sha256': digest, 'last_live_gpu': observation,
'scope': 'No physics after receipt; bounded owned cleanup then unchanged strict final GPU check'}
def filesystem_verify(binding):
# Reuse finite binding file records, not a new recursive filesystem traversal.
files = {}
def visit(x):
if isinstance(x, dict):
if isinstance(x.get('path'), str) and isinstance(x.get('sha256'), str):
files[x['path']] = x['sha256']
for item in x.values():
visit(item)
elif isinstance(x, list):
for item in x:
visit(item)
visit(binding)
files.update(binding['runtime_layout_pins'])
for filename, digest in files.items():
if b.digest(Path(filename)) != digest:
raise ValueError('identity_postexit_file_changed: ' + filename)
return {'files_rehashed': len(files), 'scope': 'static finite files; preclose same-object cfg checked by worker'}
def campaign(args):
root = args.output
is_followup = getattr(args, 'followup_v3', False)
is_skill = getattr(args, 'skill_v4', False)
if is_skill:
import skill_launch_v4 as skill
approved = skill.approval(args.approval, args.trusted_approval_sha256, root)
elif is_followup:
import functional_followup_launch_v3 as followup
approved = followup.approval(args.approval, args.trusted_approval_sha256, root)
else:
approved = launch.approval(args.approval, args.trusted_approval_sha256, root)
limits = approved['limits']
slot_rows = (skill.slots(approved['spec']) if is_skill else followup.api().slots(followup.plan()) if is_followup else
[{'slot': name, 'cycles': cycles, 'controller_kind': kind} for name, cycles, kind in launch.SLOTS])
reserved_calls = limits['max_physics_calls']
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
raise RuntimeError('Subreaper unavailable')
root.mkdir()
start = b.boottime()
deadline = min(start + limits['max_campaign_seconds'], approved['deadline_boottime']) if is_skill else start + limits['max_campaign_seconds']
campaign_guard = launch.Guard(root, start, deadline, cleanup_reserve=0,
cap=limits['max_owned_bytes'], stdout_slots=len(slot_rows))
campaign_guard.write(root / 'approval.json', approved)
if is_skill:
mailbox_path = root / 'mailbox'
mailbox_path.mkdir()
skill.mailbox().publish(mailbox_path / 'status.json', {'state': 'starting', 'global_tick': 0,
'failure': [], 'receipts_count': 0})
campaign_guard.write(root / 'reservation.json', {
'approved_by': 'current user 批准', 'start_boottime': start, 'deadline_boottime': deadline,
'limits': limits, 'reserved_physics_calls': reserved_calls, 'actual_final_calls': None,
'note': 'Reserved ceiling is not actual count; no retries/refunds/old grants.'})
policy = launch.native_policy(root, deadline, cap=limits['max_owned_bytes'], slot_names=[s['slot'] for s in slot_rows])
campaign_guard.write(root / 'native-policy.json', policy)
monitor = launch.AsyncMonitor(policy)
gpu = None
results = []
final = {'success': False, 'rows': results, 'optimizer_steps': 0, 'accepted_skill': False}
handlers = {sig: signal.signal(sig, lambda *_: setattr(campaign_guard, 'interrupted', True))
for sig in (signal.SIGTERM, signal.SIGINT)}
binding = inventory = None
try:
# Wait for one finite baseline scan, without consuming a Kit launch.
while monitor.monitor is None:
campaign_guard.check(); monitor.check(); time.sleep(.05)
if is_followup or is_skill:
from functional_gpu_watch_v3 import ObservedGPUOwnership
gpu = ObservedGPUOwnership(approved['gpu_identity'],
lambda event: campaign_guard.write(root / 'gpu-observations.json', event, cleanup=True))
else:
gpu = launch.ownership.GPUOwnership(approved['gpu_identity'])
for index, slot in enumerate(slot_rows):
name, cycles, kind = slot['slot'], slot['cycles'], slot['controller_kind']
campaign_guard.check(); monitor.check()
if is_skill:
args.skill_reservation.check_storage(args.skill_reservation.runs)
if is_followup and index >= 2 and not results[1].get('success'):
raise RuntimeError('New supervised u256 nominal three-cycle required before matrix')
if not is_followup and not is_skill and index >= 3 and not all(x.get('success') for x in results[1:3]):
raise RuntimeError('Both single cycles required before continuous runs')
output = root / name
output.mkdir()
native = root / 'native' / name
native.mkdir(parents=True)
environment = os.environ.copy()
for var in ('VIRTUAL_ENV', 'CONDA_PREFIX', 'PYTHONEXE', 'PYTHONHOME', 'args'):
environment.pop(var, None)
redirects = launch.owner.native_redirects(native, preparation=True)
for path in redirects.values():
Path(path).mkdir()
environment.update(redirects)
environment.update(PYTHONPATH=str(ROOT / 'source/SkillFactory') + ':' + str(ROOT / 'scripts'),
PYTHONUNBUFFERED='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
CUDA_VISIBLE_DEVICES='0', HEADLESS='0' if is_skill and approved['spec']['gui'] and index else '1',
LIVESTREAM='0')
if is_skill:
# Frozen import-time asset defaults predate relocatable packaging.
# Select this tree without changing their immutable source bytes.
environment.update(SKILLFACTORY_GO2_ASSET_DIR=str(ROOT / 'assets/generated/go2_fl'),
SKILLFACTORY_LEG_BUNDLE=str(ROOT / 'assets/generated/legs/go2_fl'))
slot_start = b.boottime()
seconds = (limits['max_prepare_seconds' if index == 0 else 'max_physical_seconds']
if is_followup or is_skill else limits['max_launch_seconds'])
slot_deadline = min(slot_start + seconds, deadline)
guard = launch.Guard(root, slot_start, slot_deadline, cap=limits['max_owned_bytes'], stdout_slots=len(slot_rows))
guard.check()
payload = {'mode': 'prepare' if index == 0 else 'run', 'slot': name, 'root': str(root),
'output': str(output), 'native': str(native), 'start': slot_start,
'deadline': slot_deadline, 'cycles': cycles, 'controller_kind': kind,
'sources': approved['sources'], 'launch_identity': launch.owner.launch_identity(),
'followup_v3': is_followup, 'owned_cap': limits['max_owned_bytes']}
if is_skill:
payload.update(skill_v4=True, spec=approved['spec'], mailbox=str(mailbox_path))
if is_followup:
payload.update(command_id=slot['command_id'], pair_reference=None)
if slot['pair_group'] is not None and kind == 'learned_policy':
reference = results[-1]
if not reference['success'] or reference['slot'] != slot['pair_group'] + '-reference':
raise RuntimeError('Complete matching reference required before paired u256')
path = root / reference['slot'] / 'initial-pair.json'
payload.update(pair_reference=str(path), pair_reference_sha256=b.digest(path))
if index:
payload.update(binding=str(root / 'prepare/binding.json'), binding_sha256=b.digest(root / 'prepare/binding.json'),
inventory=str(root / 'prepare/inventory.json'), inventory_sha256=b.digest(root / 'prepare/inventory.json'))
owned = launch.owner.OwnedProcesses()
gpu.check(owned, final=True)
parent, child_channel = socket.socketpair()
command = launch.owner.launch_command([str(ROOT / (skill.SCRIPT if is_skill else launch.SCRIPT)), '--worker-fd', str(child_channel.fileno())], environment)
# Persist the consumed slot BEFORE any child/AppLauncher starts.
campaign_guard.write(output / 'reservation.json', {'index': index, 'payload': payload,
'command': command, 'physics_ceiling': 0 if index == 0 else cycles * 4500 + 250})
child = capture = None
shutdown_handoff = None
clean = False
row = {'slot': name, 'success': False, 'status': 'reserved', 'pair_group': slot.get('pair_group')}
results.append(row)
try:
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, start_new_session=True,
pass_fds=(child_channel.fileno(),))
child_channel.close()
owned.attach(child)
capture = launch.owner.StdoutCapture(child.stdout, output / 'stdout.log', b)
launch.authenticate(parent, child, payload, capture, guard, monitor, owned, gpu)
parent.close()
while child.poll() is None:
campaign_guard.check(); guard.check(); monitor.check(); capture.check()
if is_skill and shutdown_handoff is None:
shutdown_handoff = release_shutdown(output, guard, gpu, owned)
if shutdown_handoff is None:
gpu.check(owned)
if is_skill and b.boottime() >= getattr(args, '_next_storage_check', 0):
args._next_storage_check = b.boottime() + 1
args.skill_reservation.check_storage(args.skill_reservation.runs)
time.sleep(.1)
capture.check()
except BaseException:
row['supervisor_error'] = traceback.format_exc()
finally:
parent.close(); child_channel.close()
if child is not None:
clean = launch.owner.cleanup_group(child, slot_deadline, b, owned,
capture.pump if capture is not None else lambda: None)
child.wait(timeout=max(.01, slot_deadline - b.boottime()))
row.update(returncode=child.returncode, cleanup_clean=clean, waited_exit=True,
owned_identities=sorted([list(x) for x in owned.known]))
if capture is not None:
capture.close()
try:
row['gpu_postexit'] = gpu.check(owned, final=True)
except BaseException:
row['gpu_postexit_error'] = traceback.format_exc()
row.setdefault('supervisor_error', row['gpu_postexit_error'])
row['end_boottime'] = b.boottime()
worker_path = output / 'worker-result.json'
if worker_path.exists():
receipt = launch.read(worker_path)
row['worker_result_sha256'] = b.digest(worker_path)
row['worker'] = receipt
row['success'] = bool(receipt.get('success') and receipt.get('preclose_identity_verified')
and row.get('returncode') == 0 and clean and not row.get('supervisor_error'))
if is_skill and index and not skill.complete_receipt(receipt, approved['spec']):
row['success'] = False
row['incomplete_worker_receipt'] = True
if is_followup and not followup.complete_receipt(receipt, slot):
row['success'] = False
row['incomplete_worker_receipt'] = True
if is_skill:
row['shutdown_handoff'] = shutdown_handoff
if (shutdown_handoff is None or not shutdown_handoff['released']
or shutdown_handoff['worker_result_sha256'] != row.get('worker_result_sha256')):
row['success'] = False
row['shutdown_handoff_missing_or_changed'] = True
if index == 0 and row['success']:
binding = launch.read(root / 'prepare/binding.json')
inventory = launch.read(root / 'prepare/inventory.json')
expected_format = ('skillfactory.go2-fl-skill-binding.v4' if is_skill else
'skillfactory.functional-followup-binding.v3' if is_followup
else 'skillfactory.contact-cycle-functional-binding.v2')
if (binding.get('format') != expected_format
or is_followup and binding.get('plan_sha256') != approved['sources'][followup.PLAN]
or binding.get('contract_sha256') != approved['sources'][launch.CONFIG]
or row['worker']['environment_constructed'] is not False
or row['worker']['physics_counts']['completed_steps'] != 0):
raise RuntimeError('Config-only parent review failed')
row['parent_review'] = filesystem_verify(binding)
# Preserve the exact prepared sources, not authority copied from old runs.
for relative in approved['sources']:
dest = root / 'source_snapshot' / relative
dest.parent.mkdir(parents=True, exist_ok=True)
b.durable_write(dest, (ROOT / relative).read_bytes())
campaign_guard.write(root / 'preparation-reviewed.json', {
'binding_sha256': b.digest(root / 'prepare/binding.json'),
'inventory_sha256': b.digest(root / 'prepare/inventory.json'),
'approved_from_current_user_budget_only': True, 'config_only_not_physical_qualification': True,
'static_verification': row['parent_review']})
elif index and row['success']:
row['postexit_files'] = filesystem_verify(binding)
if is_followup and row['success'] and slot['pair_group'] is not None and kind == 'learned_policy':
row['paired_comparison'] = followup.paired_comparison(results[-2], row)
if not row['paired_comparison']['eligible']:
row['success'] = False
if (is_followup or is_skill) and index:
row['artifact_sha256'] = {}
for filename in (('trace.pt', 'initial-state.pt', 'events-and-seams.json') if is_skill else
('trace.pt', 'initial-state.pt', 'events-and-seams.json', 'initial-pair.json')):
path = output / filename
if not path.is_file():
row['success'] = False
row.setdefault('missing_required_results', []).append(filename)
else:
row['artifact_sha256'][filename] = b.digest(path)
if is_followup and row['artifact_sha256'].get('initial-pair.json') != row.get('worker', {}).get('initial_pair_sha256'):
row['success'] = False
row['initial_pair_file_identity_mismatch'] = True
campaign_guard.write(output / 'supervisor-result.json', row)
campaign_guard.write(root / 'campaign-progress.json', final)
if not row['success']:
break # first failure consumes this campaign; never retry or tune
monitor.check()
final['success'] = len(results) == len(slot_rows) and all(x['success'] for x in results)
except BaseException:
final['error'] = traceback.format_exc()
final['success'] = False
if results:
results[-1]['success'] = False
results[-1].setdefault('supervisor_error', final['error'])
traceback.print_exc()
finally:
monitor.stop.set()
if monitor.error is not None:
final['success'] = False
final['native_monitor'] = monitor.report()
final['end_boottime'] = b.boottime()
counts = [r.get('worker', {}).get('physics_counts') for r in results]
final['actual_final_physics_calls'] = (sum(x['completed_steps'] for x in counts)
if all(x is not None for x in counts) else None)
final['reserved_physics_calls'] = reserved_calls
if final['actual_final_physics_calls'] is None or final['actual_final_physics_calls'] > reserved_calls:
final['success'] = False
final['remaining_slots'] = [s['slot'] for s in slot_rows[len(results):]]
try:
campaign_guard.write(root / 'campaign-result.json', final, cleanup=True)
if is_skill:
status = skill.mailbox().read_json(mailbox_path / 'status.json')
status.update(state='stopped' if final['success'] else 'failed',
supervisor_success=final['success'], failure=status.get('failure', []) if final['success']
else status.get('failure') or [final.get('error', 'supervision_or_worker_failure')[-4096:]])
skill.mailbox().publish(mailbox_path / 'status.json', status)
finally:
for sig, handler in handlers.items():
signal.signal(sig, handler)
return 0 if final['success'] else 2
def main():
parser = argparse.ArgumentParser(description=__doc__)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--campaign', action='store_true')
mode.add_argument('--worker-fd', type=int)
mode.add_argument('--check-followup-v3', action='store_true', help='CPU metadata only; no grant, Kit or physics')
parser.add_argument('--followup-v3', action='store_true', help='fresh u2563 then six paired command groups; new grant only')
parser.add_argument('--output', type=Path)
parser.add_argument('--approval', type=Path)
parser.add_argument('--trusted-approval-sha256')
args = parser.parse_args()
if args.check_followup_v3:
import functional_followup_launch_v3 as followup
p = followup.plan()
print(json.dumps({'execution_authorized': False, 'new_physics_calls': 0, 'optimizer_steps': 0,
'plan_sha256': b.digest(ROOT / followup.PLAN), 'plan': p,
'commands_m': followup.api().commands(p), 'slots': followup.api().slots(p),
'sources': followup.sources()}, indent=2))
return 0
if args.worker_fd is not None:
return worker(launch.worker_payload(args.worker_fd))
if args.output is None or args.approval is None or args.trusted_approval_sha256 is None:
parser.error('Require fresh independently trusted approval and new output root')
return campaign(args)
if __name__ == '__main__':
raise SystemExit(main())