#!/usr/bin/env python3 """Separately approved functional campaigns; optional frozen --followup-v3. Default: PREP + reference/u256 1/3 cycles. Followup: PREP + u2563 + six paired three-cycle groups. No training/retry or old authority. --campaign is the sole supervisor; workers require its inherited authenticated FD before AppLauncher. """ import argparse import ctypes import hashlib import importlib import json import os import signal import socket import subprocess import sys import time import traceback from dataclasses import asdict from pathlib import Path from types import SimpleNamespace import functional_flow_launch_v2 as launch ROOT, b = launch.ROOT, launch.b def actual_runtime(payload): """Config-only setup shared by PREP and RUN. Never constructs an environment.""" from isaaclab.app import AppLauncher native = Path(payload['native']) if payload.get('skill_v4'): import skill_launch_v4 gui_enabled = skill_launch_v4.gui_enabled(payload) else: gui_enabled = False gui_args = (' --/app/window/title=Go2_FL_Contact_Cycle_0.4.1 --/app/window/width=1440' ' --/app/window/height=960 --/app/window/x=120 --/app/window/y=80' ' --/app/file/ignoreUnsavedOnExit=true') if gui_enabled else '' app = AppLauncher(headless=not gui_enabled, visualizer='kit' if gui_enabled else 'none', device='cuda:0', fast_shutdown=True, kit_args=f'--/log/file={native / "kit.log"}' + gui_args).app try: # Source inventory learned in PREP is independently rehashed before RUN. inventory = None if payload['mode'] == 'prepare' else launch.read(payload['inventory'], payload['inventory_sha256']) for name in (() if inventory is None else inventory['modules']): importlib.import_module(name) task = 'SkillFactory.tasks.manager_based.contact_cycle_sim.' leg = 'SkillFactory.legs.contact_cycle_sim.' modules = {name: importlib.import_module(task + name) for name in ( 'evaluation_cfg', 'evaluation_env', 'evaluation_mdp', 'evaluation_rng_scope', 'evaluation_instrumentation', 'functional_flow_recording_v2')} modules.update({name: importlib.import_module(leg + name) for name in ( 'functional_flow_v2', 'functional_flow_binding_v2', 'evaluation_binding', 'replay')}) if payload.get('followup_v3'): for name in ('functional_followup_v3', 'functional_followup_binding_v3'): modules[name] = importlib.import_module(leg + name) importlib.import_module('functional_gpu_watch_v3') if payload.get('skill_v4'): for name in ('skill_v4', 'skill_mailbox_v4', 'skill_binding_v4', 'functional_followup_v3', 'functional_followup_binding_v3'): modules[name] = importlib.import_module(leg + name) modules['skill_recording_v4'] = importlib.import_module(task + 'skill_recording_v4') modules['performance_v042'] = importlib.import_module(task + 'performance_v042') modules['sensor_epoch_v042'] = importlib.import_module(task + 'sensor_epoch_v042') modules['packed_recording_v042'] = importlib.import_module(task + 'packed_recording_v042') modules['readonly_scope_v042'] = importlib.import_module(task + 'readonly_scope_v042') modules['batched_kinematics_v042'] = importlib.import_module(task + 'batched_kinematics_v042') modules['live_cadence_v042'] = importlib.import_module(task + 'live_cadence_v042') if payload['spec']['gui']: # PREP has no GUI experience, but must preload the exact finite # presentation dependency before inventory/binding (no viewport # construction or physics). RUN already enables this via Kit. import omni.kit.app manager = omni.kit.app.get_app().get_extension_manager() for extension in ('omni.ui', 'omni.kit.viewport.utility', 'omni.replicator.core'): if not manager.set_extension_enabled_immediate(extension, True): raise RuntimeError('gui_extension_unavailable:' + extension) modules['skill_gui_v041'] = importlib.import_module(task + 'skill_gui_v041') if payload['spec']['gui']: modules['skill_gui_v041'].prepare_native() # GUI experience preloads Replicator. Match that exact module # availability in headless PREP rather than relaxing its guard. importlib.import_module(launch.owner.OPTIONAL_REPLICATOR) for name in ('skill_launch_v4', 'functional_followup_launch_v3', 'functional_gpu_watch_v3', 'evaluate_cycle_functional_flow'): importlib.import_module(name) binding_module = modules['skill_binding_v4' if payload.get('skill_v4') else 'functional_followup_binding_v3' if payload.get('followup_v3') else 'functional_flow_binding_v2'] binder = modules['evaluation_binding'] for name in binder.REQUIRED_RUNTIME: importlib.import_module(name) instrument = modules['evaluation_instrumentation'] for name in instrument.BACKEND_SOURCE_INVENTORY: importlib.import_module(name) instrument.verify_backend_sources() replicator = launch.owner.preimport_replicator() if inventory is not None: launch.owner.check_replicator_inventory(inventory) import torch torch.cuda.init() torch.cuda.get_rng_state(torch.device('cuda:0')) if torch.cuda.get_device_name(0) != 'NVIDIA GeForce RTX 5080': raise ValueError('Require approved RTX5080') cfg = modules['evaluation_cfg'].EvaluationContactCycleEnvCfg() cfg.sim.device, cfg.seed = 'cuda:0', 93501 if payload.get('skill_v4') and payload['spec']['gui']: # Same explicit presentation config/imports in PREP and RUN. Only # RUN creates KitVisualizer/window; no physics config is changed. from isaaclab_visualizers.kit import KitVisualizerCfg importlib.import_module('isaaclab_visualizers.kit.kit_visualizer') # Observed GUI constructor-only camera helper imports must be in # PREP's exact provenance before construction, not waived afterward. importlib.import_module('isaaclab_physx.renderers.kit_viewport_utils') # Full GUI run 10: first render lazily loads this exact utility. importlib.import_module('isaaclab_physx.renderers.isaac_rtx_renderer_utils') cfg.sim.visualizer_cfgs = [KitVisualizerCfg()] if payload.get('skill_v4'): perf = modules['performance_v042'] perf.RUNTIME_MATH_QUALIFICATION = None if perf.options(ROOT)['mode'] in ('combined', 'combined_display'): nominal = torch.tensor([j['q_default'] for j in cfg.leg_bundle['spec']['joints']], device='cuda:0') perf.RUNTIME_MATH_QUALIFICATION = modules['batched_kinematics_v042'].qualify( cfg.leg_bundle, cfg.residual_parameters['reference'], nominal, 'cuda:0') launch.owner.preforward_video_recorder(cfg, modules['evaluation_env']) constructor_imports = launch.owner.preimport_constructor_modules(binder) constructor_export, export_metadata = launch.owner.physx_constructor_export(binder, resolve=True) config_imports = launch.owner.preimport_config_references(cfg, binder) binder.normalize_config(cfg) rng_scope = modules['evaluation_rng_scope'] rng_inventory = rng_scope.qualify_config(cfg) rng_inventory.update(optional_replicator=replicator, constructor_imports=constructor_imports, constructor_export=export_metadata, config_reference_imports=config_imports) instrument.qualify_backend_rng(rng_scope.backend_rng(None)) return SimpleNamespace(app=app, modules=modules, cfg=cfg, torch=torch, binding_module=binding_module, constructor_export=constructor_export, rng_inventory=rng_inventory) except BaseException: traceback.print_exc() app.close(exit_code=1) raise def verify_runtime(runtime, expected, inventory): binder = runtime.modules['evaluation_binding'] launch.owner.physx_constructor_export(binder, expected=runtime.constructor_export) launch.owner.check_replicator_inventory(inventory) return runtime.binding_module.verify(expected, ROOT, runtime.cfg, inventory) def worker(payload): root, output = Path(payload['root']), Path(payload['output']) is_followup = payload.get('followup_v3', False) is_skill = payload.get('skill_v4', False) if is_skill: import skill_launch_v4 as skill if is_followup: import functional_followup_launch_v3 as followup guard = launch.Guard(root, payload['start'], payload['deadline'], cap=payload.get('owned_cap', launch.CAP), stdout_slots=14 if is_followup else 5) interrupted = {sig: signal.signal(sig, lambda *_: setattr(guard, 'interrupted', True)) for sig in (signal.SIGTERM, signal.SIGINT)} runtime = env = hooks = recorder = gui = None result = {'success': False, 'mode': payload['mode'], 'optimizer_steps': 0, 'accepted_skill': False, 'environment_constructed': False, 'physics_counts': None, 'runtime_continuity_verified': False} expected = inventory = None code = 1 try: guard.check() if payload['sources'] != (skill.sources() if is_skill else followup.sources() if is_followup else launch.sources()) or payload['launch_identity'] != launch.owner.launch_identity(): raise ValueError('identity_worker_source_or_launch_drift') runtime = actual_runtime(payload) mod = runtime.modules if is_skill and payload['mode'] == 'run' and payload['spec']['gui']: def live_view_guard(): guard.check() mod['live_cadence_v042'].require_recorder_live(recorder) gui = mod['skill_gui_v041'].LivePresentation(runtime.app, live_view_guard, lambda value: guard.write(output / 'gui-status.json', value)) binder = runtime.binding_module hook_type = mod['evaluation_env'].RuntimeHooks if payload['mode'] == 'prepare': # Protect config preparation against any accidental task physics call. def no_physics(h): raise RuntimeError('budget_prepare_forbids_sim_step') hooks = hook_type.isaac(before_step=no_physics) hooks.__enter__() # lifetime ends only after guarded close in finally if hooks.active: inventory = binder.propose_inventory(ROOT, runtime.cfg, launch.owner.proposed_native_layouts) expected = binder.snapshot(ROOT, runtime.cfg, inventory) guard.write(output / 'inventory.json', inventory) guard.write(output / 'binding.json', expected) verify_runtime(runtime, expected, inventory) result.update(success=True, environment_constructed=False, physics_counts=mod['functional_flow_recording_v2'].counters(hooks), runtime_qualified=False, config_prepared=True, binding_sha256=b.digest(output / 'binding.json'), inventory_sha256=b.digest(output / 'inventory.json')) if is_skill: result['kit_runtime_metadata'] = mod['performance_v042'].runtime_metadata(runtime.app) else: expected = launch.read(payload['binding'], payload['binding_sha256']) inventory = launch.read(payload['inventory'], payload['inventory_sha256']) verify_runtime(runtime, expected, inventory) cfg = mod['functional_flow_v2'].load_contract(ROOT / launch.CONFIG) cycles, kind = payload['cycles'], payload['controller_kind'] if is_skill: spec = skill.validate_spec(payload['spec']) if cycles != spec['cycles'] or kind != mod['skill_v4'].CONTROLLERS[spec['controller']]: raise ValueError('Authenticated v4 slot/spec mismatch') session = mod['skill_v4'].SkillSession(spec['command'], cycles, spec['controller'], spec['queued']) protocol = mod['skill_v4'].SkillProtocol(cfg, session) result['skill_spec'] = spec elif is_followup: if cycles != 3: raise ValueError('Followup requires exactly three continuous cycles') p = mod['functional_followup_v3'].load_plan(ROOT / followup.PLAN) protocol = mod['functional_followup_v3'].FollowupProtocol(cfg, p, payload['command_id'], kind) result['command_id'] = payload['command_id'] else: protocol = mod['functional_flow_v2'].FunctionalFlowProtocol(cfg, cycles=cycles, controller_kind=kind) frozen, _ = mod['replay'].verify_bundle(ROOT) policy = (mod['replay'].load_frozen_policy(ROOT / mod['replay'].CANDIDATE, frozen).to('cuda:0') if kind == 'learned_policy' else None) # Install budget/count/reset/pose guards BEFORE environment construction. def before_physics(h): guard.check() if gui is not None and (gui.closed or not runtime.app.is_running()): raise RuntimeError('gui_window_closed_before_completion') if is_skill: mod['live_cadence_v042'].require_recorder_live(recorder) if (h.mode != 'rollout' and h.initialization_steps >= 250 or h.mode == 'rollout' and h.rollout_steps >= cycles * 4500 or h.completed_steps >= cycles * 4500 + 250): raise RuntimeError('budget_physics_call_cap') recorder = (mod['performance_v042'].recorder_class(ROOT)( cfg, protocol, guard.check, Path(payload['mailbox']), stop_at_tick=spec['stop_at_tick'], reject_at_tick=spec['reject_at_tick']) if is_skill else mod['functional_flow_recording_v2'].FunctionalFlowRecorder(cfg, protocol, guard.check)) hooks = hook_type.isaac(before_step=before_physics) hooks.__enter__() # also guard exception-path close if hooks.active: result['environment_construction_entered'] = True guard.write(output / 'worker-progress.json', result) env = mod['evaluation_env'].EvaluationContactCycleEnv( cfg=runtime.cfg, protocol=protocol, trial_index=0, runtime_hooks=hooks, evidence_callback=recorder, seam_callback=recorder.seam_callback, **({'boundary_stop_callback': lambda: session.stop_requested} if is_skill else {})) result['environment_constructed'] = True s = mod['evaluation_mdp'].state(env) if (s.robot.is_fixed_base or runtime.cfg.sim.physics.solver_type != 0 or runtime.cfg.sim.physics.enable_external_forces_every_iteration or runtime.cfg.terminations.failed is None or env.physics_dt != .002 or env.step_dt != .01 or bool(s.scenarios.any() or s.stand_only.any())): raise RuntimeError('identity_require_frozen_floating_root_nominal_failure_enabled_fixture') result['fixture'] = {'floating_root': True, 'num_envs': env.num_envs, 'system_weight_N': float(s.weight[0]), 'physics_dt_s': env.physics_dt, 'control_dt_s': env.step_dt, 'guide_enabled': runtime.cfg.guide_enabled, 'auxiliary_enabled': runtime.cfg.auxiliary_enabled} verify_runtime(runtime, expected, inventory) # constructor cannot silently mutate cfg/imports if is_skill: result['performance_installation'] = mod['performance_v042'].install_readonly_path(ROOT, env) recorder.attach(env) obs, _ = env.reset() if is_followup: pair = followup.initial_pair_record(recorder, s, obs['policy'], payload['command_id']) guard.write(output / 'initial-pair.json', pair) result['initial_pair_sha256'] = b.digest(output / 'initial-pair.json') result['initial_pair_matched'] = False if payload.get('pair_reference') is not None: expected_pair = launch.read(payload['pair_reference'], payload['pair_reference_sha256']) result['initial_pair_matched'] = followup.require_pair(pair, expected_pair) stop = env.stop_result if gui is not None: gui.start(env) if is_skill and mod['performance_v042'].options(ROOT)['mode'] == 'combined_display': mod['live_cadence_v042'].attach(recorder, env, gui) if gui.closed: stop = env._stop(('gui_window_closed_before_completion',)) controls = 0 profiler = mod['performance_v042'].LoopProfile(ROOT, env) if is_skill else None if profiler is not None: profiler.start() with runtime.torch.inference_mode(): while stop is None: guard.check() x = obs['policy'] action, mean, std = mod['replay'].inference_action( x, 'learned' if policy is not None else 'reference', policy) recorder.before_control(x, action, mean, std) stepped = env.step(action) if is_skill: # Includes the original interval20 render at the # final substep, even on the final protocol tick. mod['live_cadence_v042'].require_recorder_live(recorder) controls += 1 stop = stepped.stop obs = stepped.observation if gui is not None: gui.after_control(env) if stop is None and (gui.closed or not runtime.app.is_running()): stop = env._stop(('gui_window_closed_before_completion',)) if controls % 100 == 0 or stop is not None: guard.write(output / 'physics-counts.json', { **mod['functional_flow_recording_v2'].counters(hooks), 'physics_samples': len(recorder.frames), 'controls_entered': controls, 'stop': None if stop is None else asdict(stop)}) if stepped.reward is not None and not bool(runtime.torch.isfinite(stepped.reward).all()): raise RuntimeError('nonfinite_reward') if profiler is not None: guard.write(output / 'performance-profile.json', profiler.finish(controls, len(recorder.frames))) continuity = recorder.continuity(stop, hooks, cycles) if is_skill: result.update(live_receipts=session.receipts, pair_commands=session.pair_commands, cycles_completed=stop.completed_clock_cycles) recorder.status(env._event('tick', stop.causes), s, state='stopped', stop=stop) result.update(controller_kind=kind, cycles_requested=cycles, control_calls=controls, runtime_stop=asdict(stop), cycle_results=recorder.cycles, continuity=continuity, runtime_continuity_verified=continuity['runtime_continuity_verified'], success=continuity['runtime_continuity_verified'], physics_counts=mod['functional_flow_recording_v2'].counters(hooks)) if gui is not None: # Keep the physical scene alive, not a screenshot/trajectory # replay. No env.step after completion; close is intercepted # until evidence can be finalized by this same owner. if result['success']: recorder.status(env._event('tick', stop.causes), s, state='viewing_final', stop=stop) gui.hold(env, result['success'] and stop.reason == 'protocol_complete', safely_stopped=result['success'] and stop.reason == 'requested_stop') if is_skill: mod['live_cadence_v042'].require_recorder_live(recorder) result['gui'] = gui.report() # Close remains guarded; no reset/pose writes or extra physical steps. env.close() env = None verify_runtime(runtime, expected, inventory) result['preclose_identity_verified'] = True result['runtime'] = {'torch': str(runtime.torch.__version__), 'device': 'cuda:0', 'gpu': runtime.torch.cuda.get_device_name(0), 'rng_inventory': runtime.rng_inventory} code = 0 if result['success'] else 2 except BaseException as error: result.update(success=False, error=traceback.format_exc()) if hasattr(error, 'binding_diagnostic'): result['binding_diagnostic'] = error.binding_diagnostic traceback.print_exc() code = 1 finally: if env is not None: try: env.close() except BaseException: result['close_error'] = traceback.format_exc() result['success'], code = False, 1 if hooks is not None: hooks.__exit__(None, None, None) result['physics_counts'] = {k: getattr(hooks, k) for k in ( 'attempted_steps', 'completed_steps', 'initialization_steps', 'rollout_steps', 'reset_calls', 'pose_write_calls', 'composer_buffer_replacements')} result['hook_violations'] = hooks.violations try: if is_skill: status_path = Path(payload['mailbox']) / 'status.json' status = skill.mailbox().read_json(status_path) if not result['success']: status.update(state='failed', failure=[result.get('error', 'runtime_or_integrity_failure')[-4096:]]) status['worker_success'] = result['success'] skill.mailbox().publish(status_path, status) if recorder is not None: guard.tensor(output / 'initial-state.pt', recorder.initial, cleanup=True) guard.tensor(output / 'trace.pt', recorder.trace(), cleanup=True) guard.write(output / 'events-and-seams.json', {'events': recorder.events, 'seams': recorder.seams, 'pending_seam': recorder.pending_seam, 'recording_faults': recorder.faults, **({'live_receipts': recorder.session.receipts, 'pair_commands': recorder.session.pair_commands} if is_skill else {})}, cleanup=True) if gui is not None: result['gui'] = gui.report() result['worker_final_boottime'] = b.boottime() if is_skill: result['shutdown_handoff_ready'] = True guard.write(output / 'worker-result.json', result, cleanup=True) if is_skill: # Keep this owned GPU process alive until the supervisor has # made its last strict LIVE ownership observation. Only then # may app.close/exit race with process discovery in CLEANUP. skill.wait_shutdown_release(output, guard) except BaseException: traceback.print_exc() code = 1 for sig, handler in interrupted.items(): signal.signal(sig, handler) if gui is not None: gui.release() if runtime is not None: runtime.app.close(exit_code=code) return code def release_shutdown(output, guard, gpu, owned): """Explicit LIVE -> bounded CLEANUP transition, never an NVML tolerance. Worker waits alive after all guarded physics/env close/result publication. The original strict live check must PASS before its exit is released. Strict post-wait final GPU check remains mandatory. No unknown client is waived. """ path = output / 'worker-result.json' if not path.exists(): return None receipt = launch.read(path) if receipt.get('shutdown_handoff_ready') is not True: raise RuntimeError('process_missing_shutdown_handoff') observation = gpu.check(owned) digest = b.digest(path) guard.write(output / 'shutdown-release.json', {'worker_result_sha256': digest}) return {'released': True, 'worker_result_sha256': digest, 'last_live_gpu': observation, 'scope': 'No physics after receipt; bounded owned cleanup then unchanged strict final GPU check'} def filesystem_verify(binding): # Reuse finite binding file records, not a new recursive filesystem traversal. files = {} def visit(x): if isinstance(x, dict): if isinstance(x.get('path'), str) and isinstance(x.get('sha256'), str): files[x['path']] = x['sha256'] for item in x.values(): visit(item) elif isinstance(x, list): for item in x: visit(item) visit(binding) files.update(binding['runtime_layout_pins']) for filename, digest in files.items(): if b.digest(Path(filename)) != digest: raise ValueError('identity_postexit_file_changed: ' + filename) return {'files_rehashed': len(files), 'scope': 'static finite files; preclose same-object cfg checked by worker'} def campaign(args): root = args.output is_followup = getattr(args, 'followup_v3', False) is_skill = getattr(args, 'skill_v4', False) if is_skill: import skill_launch_v4 as skill approved = skill.approval(args.approval, args.trusted_approval_sha256, root) elif is_followup: import functional_followup_launch_v3 as followup approved = followup.approval(args.approval, args.trusted_approval_sha256, root) else: approved = launch.approval(args.approval, args.trusted_approval_sha256, root) limits = approved['limits'] slot_rows = (skill.slots(approved['spec']) if is_skill else followup.api().slots(followup.plan()) if is_followup else [{'slot': name, 'cycles': cycles, 'controller_kind': kind} for name, cycles, kind in launch.SLOTS]) reserved_calls = limits['max_physics_calls'] if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: raise RuntimeError('Subreaper unavailable') root.mkdir() start = b.boottime() deadline = min(start + limits['max_campaign_seconds'], approved['deadline_boottime']) if is_skill else start + limits['max_campaign_seconds'] campaign_guard = launch.Guard(root, start, deadline, cleanup_reserve=0, cap=limits['max_owned_bytes'], stdout_slots=len(slot_rows)) campaign_guard.write(root / 'approval.json', approved) if is_skill: mailbox_path = root / 'mailbox' mailbox_path.mkdir() skill.mailbox().publish(mailbox_path / 'status.json', {'state': 'starting', 'global_tick': 0, 'failure': [], 'receipts_count': 0}) campaign_guard.write(root / 'reservation.json', { 'approved_by': 'current user 批准', 'start_boottime': start, 'deadline_boottime': deadline, 'limits': limits, 'reserved_physics_calls': reserved_calls, 'actual_final_calls': None, 'note': 'Reserved ceiling is not actual count; no retries/refunds/old grants.'}) policy = launch.native_policy(root, deadline, cap=limits['max_owned_bytes'], slot_names=[s['slot'] for s in slot_rows]) campaign_guard.write(root / 'native-policy.json', policy) monitor = launch.AsyncMonitor(policy) gpu = None results = [] final = {'success': False, 'rows': results, 'optimizer_steps': 0, 'accepted_skill': False} handlers = {sig: signal.signal(sig, lambda *_: setattr(campaign_guard, 'interrupted', True)) for sig in (signal.SIGTERM, signal.SIGINT)} binding = inventory = None try: # Wait for one finite baseline scan, without consuming a Kit launch. while monitor.monitor is None: campaign_guard.check(); monitor.check(); time.sleep(.05) if is_followup or is_skill: from functional_gpu_watch_v3 import ObservedGPUOwnership gpu = ObservedGPUOwnership(approved['gpu_identity'], lambda event: campaign_guard.write(root / 'gpu-observations.json', event, cleanup=True)) else: gpu = launch.ownership.GPUOwnership(approved['gpu_identity']) for index, slot in enumerate(slot_rows): name, cycles, kind = slot['slot'], slot['cycles'], slot['controller_kind'] campaign_guard.check(); monitor.check() if is_skill: args.skill_reservation.check_storage(args.skill_reservation.runs) if is_followup and index >= 2 and not results[1].get('success'): raise RuntimeError('New supervised u256 nominal three-cycle required before matrix') if not is_followup and not is_skill and index >= 3 and not all(x.get('success') for x in results[1:3]): raise RuntimeError('Both single cycles required before continuous runs') output = root / name output.mkdir() native = root / 'native' / name native.mkdir(parents=True) environment = os.environ.copy() for var in ('VIRTUAL_ENV', 'CONDA_PREFIX', 'PYTHONEXE', 'PYTHONHOME', 'args'): environment.pop(var, None) redirects = launch.owner.native_redirects(native, preparation=True) for path in redirects.values(): Path(path).mkdir() environment.update(redirects) environment.update(PYTHONPATH=str(ROOT / 'source/SkillFactory') + ':' + str(ROOT / 'scripts'), PYTHONUNBUFFERED='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', CUDA_VISIBLE_DEVICES='0', HEADLESS='0' if is_skill and approved['spec']['gui'] and index else '1', LIVESTREAM='0') if is_skill: # Frozen import-time asset defaults predate relocatable packaging. # Select this tree without changing their immutable source bytes. environment.update(SKILLFACTORY_GO2_ASSET_DIR=str(ROOT / 'assets/generated/go2_fl'), SKILLFACTORY_LEG_BUNDLE=str(ROOT / 'assets/generated/legs/go2_fl')) slot_start = b.boottime() seconds = (limits['max_prepare_seconds' if index == 0 else 'max_physical_seconds'] if is_followup or is_skill else limits['max_launch_seconds']) slot_deadline = min(slot_start + seconds, deadline) guard = launch.Guard(root, slot_start, slot_deadline, cap=limits['max_owned_bytes'], stdout_slots=len(slot_rows)) guard.check() payload = {'mode': 'prepare' if index == 0 else 'run', 'slot': name, 'root': str(root), 'output': str(output), 'native': str(native), 'start': slot_start, 'deadline': slot_deadline, 'cycles': cycles, 'controller_kind': kind, 'sources': approved['sources'], 'launch_identity': launch.owner.launch_identity(), 'followup_v3': is_followup, 'owned_cap': limits['max_owned_bytes']} if is_skill: payload.update(skill_v4=True, spec=approved['spec'], mailbox=str(mailbox_path)) if is_followup: payload.update(command_id=slot['command_id'], pair_reference=None) if slot['pair_group'] is not None and kind == 'learned_policy': reference = results[-1] if not reference['success'] or reference['slot'] != slot['pair_group'] + '-reference': raise RuntimeError('Complete matching reference required before paired u256') path = root / reference['slot'] / 'initial-pair.json' payload.update(pair_reference=str(path), pair_reference_sha256=b.digest(path)) if index: payload.update(binding=str(root / 'prepare/binding.json'), binding_sha256=b.digest(root / 'prepare/binding.json'), inventory=str(root / 'prepare/inventory.json'), inventory_sha256=b.digest(root / 'prepare/inventory.json')) owned = launch.owner.OwnedProcesses() gpu.check(owned, final=True) parent, child_channel = socket.socketpair() command = launch.owner.launch_command([str(ROOT / (skill.SCRIPT if is_skill else launch.SCRIPT)), '--worker-fd', str(child_channel.fileno())], environment) # Persist the consumed slot BEFORE any child/AppLauncher starts. campaign_guard.write(output / 'reservation.json', {'index': index, 'payload': payload, 'command': command, 'physics_ceiling': 0 if index == 0 else cycles * 4500 + 250}) child = capture = None shutdown_handoff = None clean = False row = {'slot': name, 'success': False, 'status': 'reserved', 'pair_group': slot.get('pair_group')} results.append(row) try: child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True, pass_fds=(child_channel.fileno(),)) child_channel.close() owned.attach(child) capture = launch.owner.StdoutCapture(child.stdout, output / 'stdout.log', b) launch.authenticate(parent, child, payload, capture, guard, monitor, owned, gpu) parent.close() while child.poll() is None: campaign_guard.check(); guard.check(); monitor.check(); capture.check() if is_skill and shutdown_handoff is None: shutdown_handoff = release_shutdown(output, guard, gpu, owned) if shutdown_handoff is None: gpu.check(owned) if is_skill and b.boottime() >= getattr(args, '_next_storage_check', 0): args._next_storage_check = b.boottime() + 1 args.skill_reservation.check_storage(args.skill_reservation.runs) time.sleep(.1) capture.check() except BaseException: row['supervisor_error'] = traceback.format_exc() finally: parent.close(); child_channel.close() if child is not None: clean = launch.owner.cleanup_group(child, slot_deadline, b, owned, capture.pump if capture is not None else lambda: None) child.wait(timeout=max(.01, slot_deadline - b.boottime())) row.update(returncode=child.returncode, cleanup_clean=clean, waited_exit=True, owned_identities=sorted([list(x) for x in owned.known])) if capture is not None: capture.close() try: row['gpu_postexit'] = gpu.check(owned, final=True) except BaseException: row['gpu_postexit_error'] = traceback.format_exc() row.setdefault('supervisor_error', row['gpu_postexit_error']) row['end_boottime'] = b.boottime() worker_path = output / 'worker-result.json' if worker_path.exists(): receipt = launch.read(worker_path) row['worker_result_sha256'] = b.digest(worker_path) row['worker'] = receipt row['success'] = bool(receipt.get('success') and receipt.get('preclose_identity_verified') and row.get('returncode') == 0 and clean and not row.get('supervisor_error')) if is_skill and index and not skill.complete_receipt(receipt, approved['spec']): row['success'] = False row['incomplete_worker_receipt'] = True if is_followup and not followup.complete_receipt(receipt, slot): row['success'] = False row['incomplete_worker_receipt'] = True if is_skill: row['shutdown_handoff'] = shutdown_handoff if (shutdown_handoff is None or not shutdown_handoff['released'] or shutdown_handoff['worker_result_sha256'] != row.get('worker_result_sha256')): row['success'] = False row['shutdown_handoff_missing_or_changed'] = True if index == 0 and row['success']: binding = launch.read(root / 'prepare/binding.json') inventory = launch.read(root / 'prepare/inventory.json') expected_format = ('skillfactory.go2-fl-skill-binding.v4' if is_skill else 'skillfactory.functional-followup-binding.v3' if is_followup else 'skillfactory.contact-cycle-functional-binding.v2') if (binding.get('format') != expected_format or is_followup and binding.get('plan_sha256') != approved['sources'][followup.PLAN] or binding.get('contract_sha256') != approved['sources'][launch.CONFIG] or row['worker']['environment_constructed'] is not False or row['worker']['physics_counts']['completed_steps'] != 0): raise RuntimeError('Config-only parent review failed') row['parent_review'] = filesystem_verify(binding) # Preserve the exact prepared sources, not authority copied from old runs. for relative in approved['sources']: dest = root / 'source_snapshot' / relative dest.parent.mkdir(parents=True, exist_ok=True) b.durable_write(dest, (ROOT / relative).read_bytes()) campaign_guard.write(root / 'preparation-reviewed.json', { 'binding_sha256': b.digest(root / 'prepare/binding.json'), 'inventory_sha256': b.digest(root / 'prepare/inventory.json'), 'approved_from_current_user_budget_only': True, 'config_only_not_physical_qualification': True, 'static_verification': row['parent_review']}) elif index and row['success']: row['postexit_files'] = filesystem_verify(binding) if is_followup and row['success'] and slot['pair_group'] is not None and kind == 'learned_policy': row['paired_comparison'] = followup.paired_comparison(results[-2], row) if not row['paired_comparison']['eligible']: row['success'] = False if (is_followup or is_skill) and index: row['artifact_sha256'] = {} for filename in (('trace.pt', 'initial-state.pt', 'events-and-seams.json') if is_skill else ('trace.pt', 'initial-state.pt', 'events-and-seams.json', 'initial-pair.json')): path = output / filename if not path.is_file(): row['success'] = False row.setdefault('missing_required_results', []).append(filename) else: row['artifact_sha256'][filename] = b.digest(path) if is_followup and row['artifact_sha256'].get('initial-pair.json') != row.get('worker', {}).get('initial_pair_sha256'): row['success'] = False row['initial_pair_file_identity_mismatch'] = True campaign_guard.write(output / 'supervisor-result.json', row) campaign_guard.write(root / 'campaign-progress.json', final) if not row['success']: break # first failure consumes this campaign; never retry or tune monitor.check() final['success'] = len(results) == len(slot_rows) and all(x['success'] for x in results) except BaseException: final['error'] = traceback.format_exc() final['success'] = False if results: results[-1]['success'] = False results[-1].setdefault('supervisor_error', final['error']) traceback.print_exc() finally: monitor.stop.set() if monitor.error is not None: final['success'] = False final['native_monitor'] = monitor.report() final['end_boottime'] = b.boottime() counts = [r.get('worker', {}).get('physics_counts') for r in results] final['actual_final_physics_calls'] = (sum(x['completed_steps'] for x in counts) if all(x is not None for x in counts) else None) final['reserved_physics_calls'] = reserved_calls if final['actual_final_physics_calls'] is None or final['actual_final_physics_calls'] > reserved_calls: final['success'] = False final['remaining_slots'] = [s['slot'] for s in slot_rows[len(results):]] try: campaign_guard.write(root / 'campaign-result.json', final, cleanup=True) if is_skill: status = skill.mailbox().read_json(mailbox_path / 'status.json') status.update(state='stopped' if final['success'] else 'failed', supervisor_success=final['success'], failure=status.get('failure', []) if final['success'] else status.get('failure') or [final.get('error', 'supervision_or_worker_failure')[-4096:]]) skill.mailbox().publish(mailbox_path / 'status.json', status) finally: for sig, handler in handlers.items(): signal.signal(sig, handler) return 0 if final['success'] else 2 def main(): parser = argparse.ArgumentParser(description=__doc__) mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument('--campaign', action='store_true') mode.add_argument('--worker-fd', type=int) mode.add_argument('--check-followup-v3', action='store_true', help='CPU metadata only; no grant, Kit or physics') parser.add_argument('--followup-v3', action='store_true', help='fresh u2563 then six paired command groups; new grant only') parser.add_argument('--output', type=Path) parser.add_argument('--approval', type=Path) parser.add_argument('--trusted-approval-sha256') args = parser.parse_args() if args.check_followup_v3: import functional_followup_launch_v3 as followup p = followup.plan() print(json.dumps({'execution_authorized': False, 'new_physics_calls': 0, 'optimizer_steps': 0, 'plan_sha256': b.digest(ROOT / followup.PLAN), 'plan': p, 'commands_m': followup.api().commands(p), 'slots': followup.api().slots(p), 'sources': followup.sources()}, indent=2)) return 0 if args.worker_fd is not None: return worker(launch.worker_payload(args.worker_fd)) if args.output is None or args.approval is None or args.trusted_approval_sha256 is None: parser.error('Require fresh independently trusted approval and new output root') return campaign(args) if __name__ == '__main__': raise SystemExit(main())