Files
SkillFactory/scripts/audit_cycle_sim_evaluation.py
T
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

154 lines
7.2 KiB
Python

#!/usr/bin/env python3
"""CPU-only JSONL evidence re-audit; no policy, physics, registration or optimizer."""
import argparse
import importlib.util
import json
import lzma
import sys
import time
import types
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def cpu_modules():
# Private offline namespace, NOT SkillFactory and never used for frozen identity verification.
directory = ROOT / 'source/SkillFactory/SkillFactory/legs/contact_cycle_sim'
name = '_contact_cycle_offline_cpu'
package = types.ModuleType(name)
package.__path__ = [str(directory)]
sys.modules[name] = package
modules = []
for leaf in ('evaluation_protocol', 'evaluation_audit'):
spec = importlib.util.spec_from_file_location(name + '.' + leaf, directory / (leaf + '.py'))
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
modules.append(module)
return modules
def trace_lines(path, *, compressed, max_bytes, max_record_bytes, deadline, max_decoded_bytes=None):
"""Incremental legacy gzip / v2 XZ / raw JSONL, never whole-stream decompression.
XZ decoder memory is capped, including dictionaries advertised in headers.
Input, output, individual records and per-member output are independently
bounded; deadlines are checked even while draining buffered decoder output.
"""
_, limits = cpu_modules()
chunk_bytes = 64 * 1024
max_member_bytes = limits.MAX_CHUNK_RAW_BYTES
max_decoded_bytes = max_bytes * 64 if max_decoded_bytes is None else max_decoded_bytes
xz = compressed and path.name.endswith('.xz')
read_bytes = decoded_bytes = member_bytes = member_records = 0
line = bytearray()
def check():
if time.monotonic() > deadline or path.stat().st_size > max_bytes or read_bytes > max_bytes:
raise ValueError('budget_audit_resource_or_record_limit')
decoder = None
with path.open('rb') as source:
while True:
check()
data = source.read(min(chunk_bytes, max_bytes - read_bytes + 1))
read_bytes += len(data)
check() # Includes EOF and time spent reading compressed headers/tails.
if not data:
if decoder is not None:
raise EOFError('truncated compressed member')
if line:
raise ValueError('budget_audit_resource_or_record_limit')
return
# XZ may retain input internally. Drain it with bounded output before
# reading more source, including when the source is already at EOF.
while data or (xz and decoder is not None and not decoder.needs_input):
check()
member_done = False
if compressed:
if decoder is None:
decoder = (lzma.LZMADecompressor(format=lzma.FORMAT_XZ,
memlimit=limits.XZ_DECODER_MEMORY_BYTES)
if xz else zlib.decompressobj(wbits=31))
member_bytes = member_records = 0
decoded = decoder.decompress(data, chunk_bytes)
member_done = decoder.eof
if xz:
remaining = decoder.unused_data if member_done else b''
else:
remaining = decoder.unused_data if member_done else decoder.unconsumed_tail
if not decoded and len(remaining) == len(data):
raise ValueError('gzip decoder made no progress')
data = remaining
else:
decoded, data = data, b''
check()
decoded_bytes += len(decoded)
member_bytes += len(decoded)
if decoded_bytes > max_decoded_bytes or compressed and member_bytes > max_member_bytes:
raise ValueError('budget_audit_decoded_limit')
# At most one bounded record plus a 64KiB decoded block is held.
for part in decoded.splitlines(keepends=True):
line.extend(part)
if len(line) > max_record_bytes:
raise ValueError('budget_audit_resource_or_record_limit')
if line.endswith(b'\n'):
member_records += 1
if xz and member_records > limits.MAX_CHUNK_RECORDS:
raise ValueError('budget_audit_chunk_record_limit')
check()
yield bytes(line)
line.clear()
if member_done:
if xz and line:
raise ValueError('incomplete record at compressed member boundary')
decoder = None
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--protocol', type=Path, required=True)
parser.add_argument('--trial-index', type=int, required=True)
parser.add_argument('--trace', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--max-trace-bytes', type=int, required=True)
parser.add_argument('--wall-seconds', type=float, required=True)
args = parser.parse_args()
protocol_module, audit_module = cpu_modules()
protocol = protocol_module.load_protocol(args.protocol)
if not 0 < args.max_trace_bytes <= protocol.document['resources']['max_artifact_bytes']:
parser.error('Require bounded positive trace budget within protocol cap')
if not 0 < args.wall_seconds <= 3600:
parser.error('Require bounded positive CPU audit wall budget')
audit = audit_module.TrialAudit(protocol, args.trial_index)
deadline = time.monotonic() + args.wall_seconds
count = 0
try:
if args.trace.stat().st_size > args.max_trace_bytes:
raise ValueError('budget_audit_compressed_bytes')
for line in trace_lines(args.trace, compressed=args.trace.name.endswith(('.jsonl.gz', '.jsonl.xz')),
max_bytes=args.max_trace_bytes, max_record_bytes=audit_module.MAX_RECORD_BYTES,
deadline=deadline):
count += 1
if count > 2 * audit.trial.total_ticks + 16:
raise ValueError('budget_audit_resource_or_record_limit')
audit.consume(json.loads(line))
if audit.terminal is None:
audit.fail(('recording_missing_terminal',), kind='terminal')
except (OSError, EOFError, zlib.error, lzma.LZMAError, ValueError, KeyError, TypeError) as exc:
audit.fail((f'recording_offline:{exc}',), kind='offline')
report = audit.report()
report.update(terminal=audit.terminal, source_trace=str(args.trace),
new_simulation_steps=0, new_optimizer_steps=0)
with args.output.open('x') as output:
json.dump(report, output, indent=2, allow_nan=False)
output.write('\n')
print(json.dumps({'functional_passed': report['functional_passed'], 'first_failure': report['first_failure']}))
return 0 if report['functional_passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())