Files
SkillFactory/scripts/four_skills.py
T

411 lines
21 KiB
Python

#!/usr/bin/env python3
"""New v0.5.0 stage CLI: finite fresh runs, strict GPU/process ownership.
No simulation or authority is created on import. Normal callers use the public
skill service; the private worker path requires a live parent socket handoff.
"""
import argparse
import ctypes
from datetime import datetime, timezone
import fcntl
import hashlib
import json
import os
from pathlib import Path
import signal
import socket
import struct
import subprocess
import sys
import time
import traceback
from types import SimpleNamespace
ROOT = Path(__file__).resolve().parents[1]
STAGE = ROOT / 'logs/four_skills_v050'
CONFIG = ROOT / 'configs/four_skills/v050.json'
START = '2026-10-16T16:07:37+08:00'
DEADLINE = '2026-10-18T16:07:37+08:00'
def clock():
return time.clock_gettime(time.CLOCK_BOOTTIME)
def sha(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def write(path, value):
path = Path(path)
temp = path.with_suffix(path.suffix + '.pending')
data = json.dumps(value, sort_keys=True, indent=2, allow_nan=False).encode() + b'\n'
with temp.open('wb') as stream:
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
os.replace(temp, path)
def sources():
files = [p for base, pattern in [('source', '*.py'), ('scripts', '*.py'), ('configs', '*.json')]
for p in (ROOT / base).rglob(pattern) if '__pycache__' not in p.parts]
for directory in ('legs/go2_fl', 'support/go2_fl_3kg_v0', 'go2_fl'):
files += [p for p in (ROOT / 'assets/generated' / directory).rglob('*')
if p.is_file() and '__pycache__' not in p.parts]
files += [ROOT / 'docs/four_skills_v050.md', ROOT / 'demo_gui.sh', ROOT / 'demo_gui_v042.sh']
return {p.relative_to(ROOT).as_posix(): sha(p) for p in sorted(files)}
def scope():
STAGE.mkdir(exist_ok=True)
if (STAGE / 'PAUSED.json').exists():
raise PermissionError('Stage paused by user; a new explicit resume instruction is required')
p = STAGE / 'scope.json'
if not p.exists():
remaining = datetime.fromisoformat(DEADLINE).timestamp() - time.time()
value = {'format': 'skillfactory.four-skills-stage.v050', 'started_wall': START,
'deadline_wall': DEADLINE, 'boot_id': Path('/proc/sys/kernel/random/boot_id').read_text().strip(),
'deadline_boottime': clock() + remaining, 'train_cutoff_boottime': clock() + remaining - 21600,
'max_kit_launches': 64, 'max_reserved_physics_calls': 3000000,
'max_owned_bytes': 120 << 30, 'user_authorized_hours': 48,
'preserve_original_versions': True, 'scope': 'single aided Go2FL only'}
with p.open('x') as stream:
stream.write(json.dumps(value, indent=2) + '\n')
value = json.loads(p.read_text())
if value['boot_id'] != Path('/proc/sys/kernel/random/boot_id').read_text().strip():
raise RuntimeError('Stage boot changed; stop, do not revive old grants')
if clock() >= value['deadline_boottime']:
raise TimeoutError('48h stage deadline reached; no new work')
amendment = STAGE / 'scope-amendment-01.json'
if amendment.exists():
update = json.loads(amendment.read_text())
if (update.get('format') != 'skillfactory.four-skills-budget-amendment.v1'
or update.get('original_scope_sha256') != sha(p)
or update.get('max_kit_launches') != 96):
raise ValueError('Invalid recorded stage budget amendment')
value['max_kit_launches'] = 96
value['amendment_sha256'] = sha(amendment)
return value
def new_session(path, mode):
"""Explicit future caller invocation, never revival of the48h stage grant."""
global STAGE
if mode not in ('gui', 'evaluate', 'probe'):
raise PermissionError('Standalone sessions cannot authorize training')
path = Path(path)
if (not path.is_absolute() or path != path.resolve() or path.exists() or not path.parent.is_dir()
or path.is_relative_to(ROOT / 'releases') or path.is_relative_to(ROOT / 'performance_v042')
or path.is_relative_to(Path('/home/cen/sf-deliveries'))
or (path.is_relative_to(ROOT) and not path.is_relative_to(ROOT / 'logs/four_skills_v050_sessions'))
or (path.is_relative_to(Path('/home/cen/sf-runs')) and path.parent != Path('/home/cen/sf-runs'))):
raise ValueError('Fresh canonical session outside protected evidence/source trees required')
path.mkdir()
STAGE = path
value = {'format': 'skillfactory.four-skills-explicit-session.v050',
'issued_wall': datetime.now().astimezone().isoformat(),
'boot_id': Path('/proc/sys/kernel/random/boot_id').read_text().strip(),
'deadline_boottime': clock() + 3600, 'train_cutoff_boottime': clock(),
'max_kit_launches': 1, 'max_reserved_physics_calls': 50000,
'max_owned_bytes': 8 << 30, 'training_authorized': False,
'scope': 'explicit caller invocation; not a48h-task extension or old-grant reuse'}
write(path / 'scope.json', value)
return value
def tree_bytes(path):
total = 0
for base, dirs, files in os.walk(path, followlinks=False):
dirs[:] = [d for d in dirs if not Path(base, d).is_symlink()]
for name in files:
p = Path(base, name)
try:
if p.is_symlink():
raise RuntimeError('Symlink inside owned outputs')
total += p.stat().st_size
except FileNotFoundError:
continue # atomic publication disappearance, never permission denial
return total
class Guard:
def __init__(self, payload, *, worker=False):
self.p = payload
self.interrupted = False
self.next_slow = 0
self.worker = worker
def __call__(self, *, cleanup=False):
if not cleanup and (self.interrupted or (Path(self.p['output']) / 'cancel.request').exists()
or (Path(self.p['output']).parent / 'PAUSED.json').exists()):
raise InterruptedError('Cooperative cancellation')
end = self.p['deadline_boottime'] if cleanup else self.p['deadline_boottime'] - 120
if clock() >= end:
raise TimeoutError('Run/stage deadline; reserve cleanup')
if cleanup or clock() < self.next_slow:
return
self.next_slow = clock() + 1
if tree_bytes(self.p['output']) > self.p['max_owned_bytes']:
raise RuntimeError('Owned artifact cap exceeded')
if os.statvfs(self.p['output']).f_bavail * os.statvfs(self.p['output']).f_frsize < 8 << 30:
raise RuntimeError('Minimum free disk headroom8GiB')
for relative, digest in self.p['sources'].items():
if sha(ROOT / relative) != digest:
raise RuntimeError('Bound runtime source changed: ' + relative)
def reserve(args, sc):
p = STAGE / 'ledger.json'
ledger = json.loads(p.read_text()) if p.exists() else {'launches': 0, 'reserved_physics_calls': 0, 'runs': []}
calls = 160000 if args.mode == 'train' else 50000
if args.mode in ('smoke', 'probe') or args.loops == 1:
calls = 15000
if ledger['launches'] + 1 > sc['max_kit_launches'] or ledger['reserved_physics_calls'] + calls > sc['max_reserved_physics_calls']:
raise RuntimeError('Stage cap exhausted; do not silently extend')
if args.mode == 'train' and clock() >= sc['train_cutoff_boottime']:
raise TimeoutError('Final6h: no new training')
duration = 64800 if args.mode == 'train' else 2400
output = (STAGE / args.name).absolute()
if (not args.name or Path(args.name).name != args.name or args.name.startswith('.')
or output.exists() or output.parent != STAGE or output.resolve() != output):
raise ValueError('Fresh unused simple run name required')
if tree_bytes(STAGE) > sc['max_owned_bytes']:
raise RuntimeError('Stage storage cap')
checkpoint = None if args.checkpoint is None else str(args.checkpoint.resolve(strict=True))
if args.controller == 'trained' and checkpoint is None:
raise ValueError('Trained controller requires explicit immutable checkpoint')
if args.controller == 'analytic' and checkpoint is not None:
raise ValueError('Analytic controller cannot load weights')
if args.mode == 'train' and args.controller != 'analytic':
raise ValueError('Fresh training is not checkpoint resume')
output.mkdir()
payload = {'format': 'skillfactory.four-skills-run.v050', 'mode': args.mode, 'output': str(output),
'seed': args.seed, 'sequence': args.sequence, 'loops': args.loops,
'tier': args.tier, 'controller': args.controller, 'checkpoint': checkpoint,
'stop_skill': args.stop_skill or ('retract' if args.mode == 'probe' else None),
'checkpoint_sha256': None if checkpoint is None else sha(checkpoint),
'sources': sources(), 'config_sha256': sha(CONFIG), 'boot_id': sc['boot_id'],
'deadline_boottime': min(clock() + duration, sc['deadline_boottime']),
'max_physics_calls': calls, 'max_owned_bytes': (32 if args.mode == 'train' else 8) << 30,
'updates': args.updates, 'parent_pid': os.getpid(), 'issued_boottime': clock(),
'stage_amendment_sha256': sc.get('amendment_sha256')}
ledger['launches'] += 1
ledger['reserved_physics_calls'] += calls
ledger['runs'].append({'name': args.name, 'mode': args.mode, 'reserved_calls': calls,
'output': str(output), 'refunded': False})
write(p, ledger)
write(output / 'approval.json', payload)
snapshot = output / 'source_snapshot'
for relative in payload['sources']:
dest = snapshot / relative
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes((ROOT / relative).read_bytes())
return payload
def supervise(args):
import evaluate_cycle_sim_residual as owner
import rootxy_launch_v2 as ownership
from skill_launch_v4 import capture_gpu
sc = new_session(args.new_session, args.mode) if args.new_session else scope()
with (STAGE / 'serial.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
payload = reserve(args, sc)
out = Path(payload['output'])
guard = Guard(payload)
# No child command before this exact dedicated-supervisor snapshot.
owned = owner.OwnedProcesses()
gpu_identity = capture_gpu()
write(out / 'gpu-identity.json', gpu_identity)
gpu = ownership.GPUOwnership(gpu_identity)
environment = {k: v for k, v in os.environ.items() if k not in
('VIRTUAL_ENV', 'CONDA_PREFIX', 'PYTHONEXE', 'PYTHONHOME', 'args')}
environment.update(PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1',
CUDA_VISIBLE_DEVICES=gpu_identity['uuid'],
SKILLFACTORY_GO2_ASSET_DIR=str(ROOT / 'assets/generated/go2_fl'),
SKILLFACTORY_LEG_BUNDLE=str(ROOT / 'assets/generated/legs/go2_fl'),
PYTHONPATH=os.pathsep.join([str(ROOT / 'source/SkillFactory'), str(ROOT / 'scripts'),
environment.get('PYTHONPATH', '')]))
(out / 'native').mkdir()
redirects = owner.native_redirects(out / 'native' / 'runtime', preparation=True)
for p in redirects.values():
Path(p).mkdir(parents=True, exist_ok=True)
environment.update(redirects)
policy = {'artifact_root': str(out), 'mount_identity': owner.mount_identity(out),
'native_paths': owner.native_paths(out), 'shared_native_paths': owner.shared_native_paths(),
'task_owned_cache_paths': sorted(redirects.values()), 'free_space_paths': owner.free_space_paths(out),
'free_space_mount_identities': {p: owner.mount_identity(Path(p)) for p in owner.free_space_paths(out)},
'check_interval_seconds': 1., 'min_free_bytes': 8 << 30,
'max_native_growth_bytes': 16 << 30}
write(out / 'native-policy.json', policy)
monitor = owner.NativeMonitor(policy, SimpleNamespace(boottime=clock, STORAGE_SCOPE='four-skills-v050-monitored-not-quota'))
child = capture = None
errors = []
clean = waited = False
parent, channel = socket.socketpair()
old_handlers = {}
try:
if ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) != 0:
raise RuntimeError('Subreaper setup failed')
for sig in (signal.SIGTERM, signal.SIGINT):
old_handlers[sig] = signal.signal(sig, lambda *_: setattr(guard, 'interrupted', True))
command = owner.launch_command([str(Path(__file__).resolve()), '--_worker-fd', str(channel.fileno())], environment)
write(out / 'invocation.json', {'command': command, 'cwd': str(ROOT), 'launch_identity': owner.launch_identity()})
child = subprocess.Popen(command, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, start_new_session=True, pass_fds=(channel.fileno(),))
owned.attach(child)
channel.close()
capture = owner.StdoutCapture(child.stdout, out / 'stdout.log', SimpleNamespace(STDOUT_BYTES=8 << 20))
packet = json.dumps(payload).encode()
parent.sendall(struct.pack('!I', len(packet)) + packet)
parent.close()
next_heartbeat = 0
while child.poll() is None or not capture.eof:
guard()
capture.check()
gpu.check(owned)
monitor.check()
if clock() >= next_heartbeat:
write(out / 'supervisor-heartbeat.json', {'boottime': clock(), 'owned': sorted(owned.known),
'gpu': gpu.clients().__str__(), 'deadline': payload['deadline_boottime']})
next_heartbeat = clock() + 5
time.sleep(.05)
child.wait()
waited = True
if child.returncode != 0:
errors.append('worker_nonzero:' + str(child.returncode))
except BaseException:
errors.append(traceback.format_exc())
finally:
parent.close()
channel.close()
try:
if child is not None:
if child.poll() is None and errors:
# File cancellation survives SDK signal-handler changes;
# allow Python evidence/checkpoint finalizers first.
write(out / 'cancel.request', {'reason': errors[-1][-4096:]})
until = min(clock() + 15, payload['deadline_boottime'] - 10)
while child.poll() is None and clock() < until:
if capture is not None:
capture.pump()
time.sleep(.05)
clean = owner.cleanup_group(child, payload['deadline_boottime'], SimpleNamespace(boottime=clock),
owned, lambda: capture.pump() if capture else None)
child.wait(timeout=max(.1, payload['deadline_boottime'] - clock()))
waited = True
if capture is not None:
capture.close()
if capture.discarded or capture.error or not capture.eof:
errors.append('incomplete_stdout')
final_gpu = gpu.check(owned, final=True)
guard(cleanup=True)
monitor.check(force=True)
except BaseException:
errors.append('cleanup:' + traceback.format_exc())
final_gpu = None
worker = json.loads((out / 'worker-result.json').read_text()) if (out / 'worker-result.json').exists() else None
if worker is not None and not worker.get('success'):
errors.append('worker_reported_failure:' + str(worker.get('error', worker.get('close_error', 'incomplete')))[-4096:])
success = bool(not errors and clean and waited and worker and worker.get('success')
and worker.get('postclose_identity_verified') and child.returncode == 0)
result = {'success': success, 'functional_success': bool(success and worker.get('functional_success')),
'training_complete': bool(success and worker.get('training_complete')),
'owned_cleanup': clean, 'waited_exit': waited,
'returncode': None if child is None else child.returncode, 'errors': errors,
'gpu_final': final_gpu, 'owned_identities': sorted(owned.known),
'native_monitor': monitor.report(), 'accepted_skill': False}
write(out / 'supervisor-result.json', result)
for sig, handler in old_handlers.items():
signal.signal(sig, handler)
print(json.dumps({'output': str(out), 'success': success,
'functional_success': result['functional_success'],
'training_complete': result['training_complete'], 'errors': errors}), flush=True)
return 0 if success else 1
def supervisor_command_matches(command, cwd):
if len(command) < 3:
return False
try:
script = Path(command[1].decode())
if not script.is_absolute():
script = Path(cwd) / script
return (script.resolve() == Path(__file__).resolve()
and command[2].decode() in ('smoke', 'evaluate', 'train', 'gui', 'probe'))
except (UnicodeError, OSError, ValueError):
return False
def worker(fd):
channel = socket.socket(fileno=fd)
peer_pid, peer_uid, _ = struct.unpack('3i', channel.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
import evaluate_cycle_sim_residual as owner
command = Path(f'/proc/{peer_pid}/cmdline').read_bytes().split(b'\0')
if (peer_uid != os.getuid() or not owner.ancestor(peer_pid, os.getpid())
or Path(f'/proc/{peer_pid}/exe').resolve() != Path('/usr/bin/python3.12')
or not supervisor_command_matches(command, Path(f'/proc/{peer_pid}/cwd').resolve())):
raise PermissionError('Authenticated exact live sole ancestor supervisor required')
channel.settimeout(30)
def receive(n):
chunks = b''
while len(chunks) < n:
part = channel.recv(n - len(chunks))
if not part:
raise EOFError('Incomplete parent handoff')
chunks += part
return chunks
size = struct.unpack('!I', receive(4))[0]
if size > 4 << 20:
raise ValueError('Oversized handoff')
payload = json.loads(receive(size))
channel.close()
if payload['parent_pid'] != peer_pid or payload['format'] != 'skillfactory.four-skills-run.v050':
raise PermissionError('Wrong run identity')
if payload != json.loads((Path(payload['output']) / 'approval.json').read_text()):
raise PermissionError('Exact new approval required')
claim = Path(payload['output']) / 'worker-claim'
with claim.open('x'):
pass
guard = Guard(payload, worker=True)
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: setattr(guard, 'interrupted', True))
guard()
from four_skills_runtime import run
return run(payload, guard, write)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--_worker-fd', type=int, help=argparse.SUPPRESS)
parser.add_argument('mode', nargs='?', choices=('smoke', 'evaluate', 'train', 'gui', 'probe', 'scope'))
parser.add_argument('--name')
parser.add_argument('--new-session', type=Path,
help='Explicit fresh<=1h nontraining invocation; no old stage-grant reuse')
parser.add_argument('--controller', choices=('analytic', 'trained'), default='analytic')
parser.add_argument('--checkpoint', type=Path)
parser.add_argument('--tier', choices=('none', 'small', 'medium', 'full'), default='none')
parser.add_argument('--seed', type=int, default=115011)
parser.add_argument('--sequence', type=int, choices=(0, 1, 2), default=0)
parser.add_argument('--loops', type=int, choices=(1, 2, 3, 4, 5))
parser.add_argument('--updates', type=int)
parser.add_argument('--stop-skill', choices=('retract', 'swing'))
args = parser.parse_args()
if args._worker_fd is not None:
return worker(args._worker_fd)
if args.mode == 'scope':
print(json.dumps(scope(), indent=2))
return 0
if args.mode is None or args.name is None:
parser.error('Fresh --name and run mode required')
if args.mode == 'train':
args.seed = 105011
if args.updates is not None and not 1 <= args.updates <= 256:
parser.error('Updates in1..256')
return supervise(args)
if __name__ == '__main__':
raise SystemExit(main())