411 lines
21 KiB
Python
411 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""New v0.5.0 stage CLI: finite fresh runs, strict GPU/process ownership.
|
|
|
|
No simulation or authority is created on import. Normal callers use the public
|
|
skill service; the private worker path requires a live parent socket handoff.
|
|
"""
|
|
import argparse
|
|
import ctypes
|
|
from datetime import datetime, timezone
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import socket
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from types import SimpleNamespace
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
STAGE = ROOT / 'logs/four_skills_v050'
|
|
CONFIG = ROOT / 'configs/four_skills/v050.json'
|
|
START = '2026-10-16T16:07:37+08:00'
|
|
DEADLINE = '2026-10-18T16:07:37+08:00'
|
|
|
|
|
|
def clock():
|
|
return time.clock_gettime(time.CLOCK_BOOTTIME)
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def write(path, value):
|
|
path = Path(path)
|
|
temp = path.with_suffix(path.suffix + '.pending')
|
|
data = json.dumps(value, sort_keys=True, indent=2, allow_nan=False).encode() + b'\n'
|
|
with temp.open('wb') as stream:
|
|
stream.write(data)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temp, path)
|
|
|
|
|
|
def sources():
|
|
files = [p for base, pattern in [('source', '*.py'), ('scripts', '*.py'), ('configs', '*.json')]
|
|
for p in (ROOT / base).rglob(pattern) if '__pycache__' not in p.parts]
|
|
for directory in ('legs/go2_fl', 'support/go2_fl_3kg_v0', 'go2_fl'):
|
|
files += [p for p in (ROOT / 'assets/generated' / directory).rglob('*')
|
|
if p.is_file() and '__pycache__' not in p.parts]
|
|
files += [ROOT / 'docs/four_skills_v050.md', ROOT / 'demo_gui.sh', ROOT / 'demo_gui_v042.sh']
|
|
return {p.relative_to(ROOT).as_posix(): sha(p) for p in sorted(files)}
|
|
|
|
|
|
def scope():
|
|
STAGE.mkdir(exist_ok=True)
|
|
if (STAGE / 'PAUSED.json').exists():
|
|
raise PermissionError('Stage paused by user; a new explicit resume instruction is required')
|
|
p = STAGE / 'scope.json'
|
|
if not p.exists():
|
|
remaining = datetime.fromisoformat(DEADLINE).timestamp() - time.time()
|
|
value = {'format': 'skillfactory.four-skills-stage.v050', 'started_wall': START,
|
|
'deadline_wall': DEADLINE, 'boot_id': Path('/proc/sys/kernel/random/boot_id').read_text().strip(),
|
|
'deadline_boottime': clock() + remaining, 'train_cutoff_boottime': clock() + remaining - 21600,
|
|
'max_kit_launches': 64, 'max_reserved_physics_calls': 3000000,
|
|
'max_owned_bytes': 120 << 30, 'user_authorized_hours': 48,
|
|
'preserve_original_versions': True, 'scope': 'single aided Go2FL only'}
|
|
with p.open('x') as stream:
|
|
stream.write(json.dumps(value, indent=2) + '\n')
|
|
value = json.loads(p.read_text())
|
|
if value['boot_id'] != Path('/proc/sys/kernel/random/boot_id').read_text().strip():
|
|
raise RuntimeError('Stage boot changed; stop, do not revive old grants')
|
|
if clock() >= value['deadline_boottime']:
|
|
raise TimeoutError('48h stage deadline reached; no new work')
|
|
amendment = STAGE / 'scope-amendment-01.json'
|
|
if amendment.exists():
|
|
update = json.loads(amendment.read_text())
|
|
if (update.get('format') != 'skillfactory.four-skills-budget-amendment.v1'
|
|
or update.get('original_scope_sha256') != sha(p)
|
|
or update.get('max_kit_launches') != 96):
|
|
raise ValueError('Invalid recorded stage budget amendment')
|
|
value['max_kit_launches'] = 96
|
|
value['amendment_sha256'] = sha(amendment)
|
|
return value
|
|
|
|
|
|
def new_session(path, mode):
|
|
"""Explicit future caller invocation, never revival of the48h stage grant."""
|
|
global STAGE
|
|
if mode not in ('gui', 'evaluate', 'probe'):
|
|
raise PermissionError('Standalone sessions cannot authorize training')
|
|
path = Path(path)
|
|
if (not path.is_absolute() or path != path.resolve() or path.exists() or not path.parent.is_dir()
|
|
or path.is_relative_to(ROOT / 'releases') or path.is_relative_to(ROOT / 'performance_v042')
|
|
or path.is_relative_to(Path('/home/cen/sf-deliveries'))
|
|
or (path.is_relative_to(ROOT) and not path.is_relative_to(ROOT / 'logs/four_skills_v050_sessions'))
|
|
or (path.is_relative_to(Path('/home/cen/sf-runs')) and path.parent != Path('/home/cen/sf-runs'))):
|
|
raise ValueError('Fresh canonical session outside protected evidence/source trees required')
|
|
path.mkdir()
|
|
STAGE = path
|
|
value = {'format': 'skillfactory.four-skills-explicit-session.v050',
|
|
'issued_wall': datetime.now().astimezone().isoformat(),
|
|
'boot_id': Path('/proc/sys/kernel/random/boot_id').read_text().strip(),
|
|
'deadline_boottime': clock() + 3600, 'train_cutoff_boottime': clock(),
|
|
'max_kit_launches': 1, 'max_reserved_physics_calls': 50000,
|
|
'max_owned_bytes': 8 << 30, 'training_authorized': False,
|
|
'scope': 'explicit caller invocation; not a48h-task extension or old-grant reuse'}
|
|
write(path / 'scope.json', value)
|
|
return value
|
|
|
|
|
|
def tree_bytes(path):
|
|
total = 0
|
|
for base, dirs, files in os.walk(path, followlinks=False):
|
|
dirs[:] = [d for d in dirs if not Path(base, d).is_symlink()]
|
|
for name in files:
|
|
p = Path(base, name)
|
|
try:
|
|
if p.is_symlink():
|
|
raise RuntimeError('Symlink inside owned outputs')
|
|
total += p.stat().st_size
|
|
except FileNotFoundError:
|
|
continue # atomic publication disappearance, never permission denial
|
|
return total
|
|
|
|
|
|
class Guard:
|
|
def __init__(self, payload, *, worker=False):
|
|
self.p = payload
|
|
self.interrupted = False
|
|
self.next_slow = 0
|
|
self.worker = worker
|
|
|
|
def __call__(self, *, cleanup=False):
|
|
if not cleanup and (self.interrupted or (Path(self.p['output']) / 'cancel.request').exists()
|
|
or (Path(self.p['output']).parent / 'PAUSED.json').exists()):
|
|
raise InterruptedError('Cooperative cancellation')
|
|
end = self.p['deadline_boottime'] if cleanup else self.p['deadline_boottime'] - 120
|
|
if clock() >= end:
|
|
raise TimeoutError('Run/stage deadline; reserve cleanup')
|
|
if cleanup or clock() < self.next_slow:
|
|
return
|
|
self.next_slow = clock() + 1
|
|
if tree_bytes(self.p['output']) > self.p['max_owned_bytes']:
|
|
raise RuntimeError('Owned artifact cap exceeded')
|
|
if os.statvfs(self.p['output']).f_bavail * os.statvfs(self.p['output']).f_frsize < 8 << 30:
|
|
raise RuntimeError('Minimum free disk headroom8GiB')
|
|
for relative, digest in self.p['sources'].items():
|
|
if sha(ROOT / relative) != digest:
|
|
raise RuntimeError('Bound runtime source changed: ' + relative)
|
|
|
|
|
|
def reserve(args, sc):
|
|
p = STAGE / 'ledger.json'
|
|
ledger = json.loads(p.read_text()) if p.exists() else {'launches': 0, 'reserved_physics_calls': 0, 'runs': []}
|
|
calls = 160000 if args.mode == 'train' else 50000
|
|
if args.mode in ('smoke', 'probe') or args.loops == 1:
|
|
calls = 15000
|
|
if ledger['launches'] + 1 > sc['max_kit_launches'] or ledger['reserved_physics_calls'] + calls > sc['max_reserved_physics_calls']:
|
|
raise RuntimeError('Stage cap exhausted; do not silently extend')
|
|
if args.mode == 'train' and clock() >= sc['train_cutoff_boottime']:
|
|
raise TimeoutError('Final6h: no new training')
|
|
duration = 64800 if args.mode == 'train' else 2400
|
|
output = (STAGE / args.name).absolute()
|
|
if (not args.name or Path(args.name).name != args.name or args.name.startswith('.')
|
|
or output.exists() or output.parent != STAGE or output.resolve() != output):
|
|
raise ValueError('Fresh unused simple run name required')
|
|
if tree_bytes(STAGE) > sc['max_owned_bytes']:
|
|
raise RuntimeError('Stage storage cap')
|
|
checkpoint = None if args.checkpoint is None else str(args.checkpoint.resolve(strict=True))
|
|
if args.controller == 'trained' and checkpoint is None:
|
|
raise ValueError('Trained controller requires explicit immutable checkpoint')
|
|
if args.controller == 'analytic' and checkpoint is not None:
|
|
raise ValueError('Analytic controller cannot load weights')
|
|
if args.mode == 'train' and args.controller != 'analytic':
|
|
raise ValueError('Fresh training is not checkpoint resume')
|
|
output.mkdir()
|
|
payload = {'format': 'skillfactory.four-skills-run.v050', 'mode': args.mode, 'output': str(output),
|
|
'seed': args.seed, 'sequence': args.sequence, 'loops': args.loops,
|
|
'tier': args.tier, 'controller': args.controller, 'checkpoint': checkpoint,
|
|
'stop_skill': args.stop_skill or ('retract' if args.mode == 'probe' else None),
|
|
'checkpoint_sha256': None if checkpoint is None else sha(checkpoint),
|
|
'sources': sources(), 'config_sha256': sha(CONFIG), 'boot_id': sc['boot_id'],
|
|
'deadline_boottime': min(clock() + duration, sc['deadline_boottime']),
|
|
'max_physics_calls': calls, 'max_owned_bytes': (32 if args.mode == 'train' else 8) << 30,
|
|
'updates': args.updates, 'parent_pid': os.getpid(), 'issued_boottime': clock(),
|
|
'stage_amendment_sha256': sc.get('amendment_sha256')}
|
|
ledger['launches'] += 1
|
|
ledger['reserved_physics_calls'] += calls
|
|
ledger['runs'].append({'name': args.name, 'mode': args.mode, 'reserved_calls': calls,
|
|
'output': str(output), 'refunded': False})
|
|
write(p, ledger)
|
|
write(output / 'approval.json', payload)
|
|
snapshot = output / 'source_snapshot'
|
|
for relative in payload['sources']:
|
|
dest = snapshot / relative
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes((ROOT / relative).read_bytes())
|
|
return payload
|
|
|
|
|
|
def supervise(args):
|
|
import evaluate_cycle_sim_residual as owner
|
|
import rootxy_launch_v2 as ownership
|
|
from skill_launch_v4 import capture_gpu
|
|
sc = new_session(args.new_session, args.mode) if args.new_session else scope()
|
|
with (STAGE / 'serial.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
payload = reserve(args, sc)
|
|
out = Path(payload['output'])
|
|
guard = Guard(payload)
|
|
# No child command before this exact dedicated-supervisor snapshot.
|
|
owned = owner.OwnedProcesses()
|
|
gpu_identity = capture_gpu()
|
|
write(out / 'gpu-identity.json', gpu_identity)
|
|
gpu = ownership.GPUOwnership(gpu_identity)
|
|
environment = {k: v for k, v in os.environ.items() if k not in
|
|
('VIRTUAL_ENV', 'CONDA_PREFIX', 'PYTHONEXE', 'PYTHONHOME', 'args')}
|
|
environment.update(PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
|
|
CUDA_VISIBLE_DEVICES=gpu_identity['uuid'],
|
|
SKILLFACTORY_GO2_ASSET_DIR=str(ROOT / 'assets/generated/go2_fl'),
|
|
SKILLFACTORY_LEG_BUNDLE=str(ROOT / 'assets/generated/legs/go2_fl'),
|
|
PYTHONPATH=os.pathsep.join([str(ROOT / 'source/SkillFactory'), str(ROOT / 'scripts'),
|
|
environment.get('PYTHONPATH', '')]))
|
|
(out / 'native').mkdir()
|
|
redirects = owner.native_redirects(out / 'native' / 'runtime', preparation=True)
|
|
for p in redirects.values():
|
|
Path(p).mkdir(parents=True, exist_ok=True)
|
|
environment.update(redirects)
|
|
policy = {'artifact_root': str(out), 'mount_identity': owner.mount_identity(out),
|
|
'native_paths': owner.native_paths(out), 'shared_native_paths': owner.shared_native_paths(),
|
|
'task_owned_cache_paths': sorted(redirects.values()), 'free_space_paths': owner.free_space_paths(out),
|
|
'free_space_mount_identities': {p: owner.mount_identity(Path(p)) for p in owner.free_space_paths(out)},
|
|
'check_interval_seconds': 1., 'min_free_bytes': 8 << 30,
|
|
'max_native_growth_bytes': 16 << 30}
|
|
write(out / 'native-policy.json', policy)
|
|
monitor = owner.NativeMonitor(policy, SimpleNamespace(boottime=clock, STORAGE_SCOPE='four-skills-v050-monitored-not-quota'))
|
|
child = capture = None
|
|
errors = []
|
|
clean = waited = False
|
|
parent, channel = socket.socketpair()
|
|
old_handlers = {}
|
|
try:
|
|
if ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) != 0:
|
|
raise RuntimeError('Subreaper setup failed')
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
old_handlers[sig] = signal.signal(sig, lambda *_: setattr(guard, 'interrupted', True))
|
|
command = owner.launch_command([str(Path(__file__).resolve()), '--_worker-fd', str(channel.fileno())], environment)
|
|
write(out / 'invocation.json', {'command': command, 'cwd': str(ROOT), 'launch_identity': owner.launch_identity()})
|
|
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, start_new_session=True, pass_fds=(channel.fileno(),))
|
|
owned.attach(child)
|
|
channel.close()
|
|
capture = owner.StdoutCapture(child.stdout, out / 'stdout.log', SimpleNamespace(STDOUT_BYTES=8 << 20))
|
|
packet = json.dumps(payload).encode()
|
|
parent.sendall(struct.pack('!I', len(packet)) + packet)
|
|
parent.close()
|
|
next_heartbeat = 0
|
|
while child.poll() is None or not capture.eof:
|
|
guard()
|
|
capture.check()
|
|
gpu.check(owned)
|
|
monitor.check()
|
|
if clock() >= next_heartbeat:
|
|
write(out / 'supervisor-heartbeat.json', {'boottime': clock(), 'owned': sorted(owned.known),
|
|
'gpu': gpu.clients().__str__(), 'deadline': payload['deadline_boottime']})
|
|
next_heartbeat = clock() + 5
|
|
time.sleep(.05)
|
|
child.wait()
|
|
waited = True
|
|
if child.returncode != 0:
|
|
errors.append('worker_nonzero:' + str(child.returncode))
|
|
except BaseException:
|
|
errors.append(traceback.format_exc())
|
|
finally:
|
|
parent.close()
|
|
channel.close()
|
|
try:
|
|
if child is not None:
|
|
if child.poll() is None and errors:
|
|
# File cancellation survives SDK signal-handler changes;
|
|
# allow Python evidence/checkpoint finalizers first.
|
|
write(out / 'cancel.request', {'reason': errors[-1][-4096:]})
|
|
until = min(clock() + 15, payload['deadline_boottime'] - 10)
|
|
while child.poll() is None and clock() < until:
|
|
if capture is not None:
|
|
capture.pump()
|
|
time.sleep(.05)
|
|
clean = owner.cleanup_group(child, payload['deadline_boottime'], SimpleNamespace(boottime=clock),
|
|
owned, lambda: capture.pump() if capture else None)
|
|
child.wait(timeout=max(.1, payload['deadline_boottime'] - clock()))
|
|
waited = True
|
|
if capture is not None:
|
|
capture.close()
|
|
if capture.discarded or capture.error or not capture.eof:
|
|
errors.append('incomplete_stdout')
|
|
final_gpu = gpu.check(owned, final=True)
|
|
guard(cleanup=True)
|
|
monitor.check(force=True)
|
|
except BaseException:
|
|
errors.append('cleanup:' + traceback.format_exc())
|
|
final_gpu = None
|
|
worker = json.loads((out / 'worker-result.json').read_text()) if (out / 'worker-result.json').exists() else None
|
|
if worker is not None and not worker.get('success'):
|
|
errors.append('worker_reported_failure:' + str(worker.get('error', worker.get('close_error', 'incomplete')))[-4096:])
|
|
success = bool(not errors and clean and waited and worker and worker.get('success')
|
|
and worker.get('postclose_identity_verified') and child.returncode == 0)
|
|
result = {'success': success, 'functional_success': bool(success and worker.get('functional_success')),
|
|
'training_complete': bool(success and worker.get('training_complete')),
|
|
'owned_cleanup': clean, 'waited_exit': waited,
|
|
'returncode': None if child is None else child.returncode, 'errors': errors,
|
|
'gpu_final': final_gpu, 'owned_identities': sorted(owned.known),
|
|
'native_monitor': monitor.report(), 'accepted_skill': False}
|
|
write(out / 'supervisor-result.json', result)
|
|
for sig, handler in old_handlers.items():
|
|
signal.signal(sig, handler)
|
|
print(json.dumps({'output': str(out), 'success': success,
|
|
'functional_success': result['functional_success'],
|
|
'training_complete': result['training_complete'], 'errors': errors}), flush=True)
|
|
return 0 if success else 1
|
|
|
|
|
|
def supervisor_command_matches(command, cwd):
|
|
if len(command) < 3:
|
|
return False
|
|
try:
|
|
script = Path(command[1].decode())
|
|
if not script.is_absolute():
|
|
script = Path(cwd) / script
|
|
return (script.resolve() == Path(__file__).resolve()
|
|
and command[2].decode() in ('smoke', 'evaluate', 'train', 'gui', 'probe'))
|
|
except (UnicodeError, OSError, ValueError):
|
|
return False
|
|
|
|
|
|
def worker(fd):
|
|
channel = socket.socket(fileno=fd)
|
|
peer_pid, peer_uid, _ = struct.unpack('3i', channel.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
|
|
import evaluate_cycle_sim_residual as owner
|
|
command = Path(f'/proc/{peer_pid}/cmdline').read_bytes().split(b'\0')
|
|
if (peer_uid != os.getuid() or not owner.ancestor(peer_pid, os.getpid())
|
|
or Path(f'/proc/{peer_pid}/exe').resolve() != Path('/usr/bin/python3.12')
|
|
or not supervisor_command_matches(command, Path(f'/proc/{peer_pid}/cwd').resolve())):
|
|
raise PermissionError('Authenticated exact live sole ancestor supervisor required')
|
|
channel.settimeout(30)
|
|
def receive(n):
|
|
chunks = b''
|
|
while len(chunks) < n:
|
|
part = channel.recv(n - len(chunks))
|
|
if not part:
|
|
raise EOFError('Incomplete parent handoff')
|
|
chunks += part
|
|
return chunks
|
|
size = struct.unpack('!I', receive(4))[0]
|
|
if size > 4 << 20:
|
|
raise ValueError('Oversized handoff')
|
|
payload = json.loads(receive(size))
|
|
channel.close()
|
|
if payload['parent_pid'] != peer_pid or payload['format'] != 'skillfactory.four-skills-run.v050':
|
|
raise PermissionError('Wrong run identity')
|
|
if payload != json.loads((Path(payload['output']) / 'approval.json').read_text()):
|
|
raise PermissionError('Exact new approval required')
|
|
claim = Path(payload['output']) / 'worker-claim'
|
|
with claim.open('x'):
|
|
pass
|
|
guard = Guard(payload, worker=True)
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
signal.signal(sig, lambda *_: setattr(guard, 'interrupted', True))
|
|
guard()
|
|
from four_skills_runtime import run
|
|
return run(payload, guard, write)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--_worker-fd', type=int, help=argparse.SUPPRESS)
|
|
parser.add_argument('mode', nargs='?', choices=('smoke', 'evaluate', 'train', 'gui', 'probe', 'scope'))
|
|
parser.add_argument('--name')
|
|
parser.add_argument('--new-session', type=Path,
|
|
help='Explicit fresh<=1h nontraining invocation; no old stage-grant reuse')
|
|
parser.add_argument('--controller', choices=('analytic', 'trained'), default='analytic')
|
|
parser.add_argument('--checkpoint', type=Path)
|
|
parser.add_argument('--tier', choices=('none', 'small', 'medium', 'full'), default='none')
|
|
parser.add_argument('--seed', type=int, default=115011)
|
|
parser.add_argument('--sequence', type=int, choices=(0, 1, 2), default=0)
|
|
parser.add_argument('--loops', type=int, choices=(1, 2, 3, 4, 5))
|
|
parser.add_argument('--updates', type=int)
|
|
parser.add_argument('--stop-skill', choices=('retract', 'swing'))
|
|
args = parser.parse_args()
|
|
if args._worker_fd is not None:
|
|
return worker(args._worker_fd)
|
|
if args.mode == 'scope':
|
|
print(json.dumps(scope(), indent=2))
|
|
return 0
|
|
if args.mode is None or args.name is None:
|
|
parser.error('Fresh --name and run mode required')
|
|
if args.mode == 'train':
|
|
args.seed = 105011
|
|
if args.updates is not None and not 1 <= args.updates <= 256:
|
|
parser.error('Updates in1..256')
|
|
return supervise(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|