Files
SkillFactory/scripts/profile_cycle_sim_evidence.py
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

198 lines
10 KiB
Python

#!/usr/bin/env python3
"""CPU-only saved-record component replay; no simulator, inference or training.
Private namespace packages bypass task registration, for offline measurement only,
never for identity verification. Originals are read-only; outputs are exclusive.
"""
import argparse
import cProfile
import dataclasses
import gzip
import hashlib
import importlib.util
import json
import lzma
import pstats
import sys
import time
import types
from collections import defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def modules():
base = ROOT / 'source/SkillFactory/SkillFactory'
for relative in ('', '.legs', '.legs.contact_cycle', '.legs.contact_cycle_sim'):
name = 'SkillFactory' + relative
package = types.ModuleType(name)
package.__path__ = [str(base / relative.lstrip('.').replace('.', '/'))]
sys.modules[name] = package
from SkillFactory.legs.contact_cycle_sim import evaluation_audit, evaluation_budget, evaluation_protocol
path = base / 'tasks/manager_based/contact_cycle_sim/evaluation_recording.py'
spec = importlib.util.spec_from_file_location('_offline_recording', path)
recording = importlib.util.module_from_spec(spec)
spec.loader.exec_module(recording)
return evaluation_protocol, evaluation_audit, evaluation_budget, recording
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--row0', type=Path, required=True)
parser.add_argument('--row1', type=Path, required=True)
parser.add_argument('--budget-root', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--replay-directory', type=Path)
args = parser.parse_args()
protocol_module, audit_module, budget_module, recording = modules()
protocol = protocol_module.load_protocol(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json')
result = {'scope': 'CPU saved-record replay only; not end-to-end or GPU throughput',
'python': sys.version, 'rows': [], 'budget_root': str(args.budget_root)}
deadline = time.monotonic() + 1800
if args.replay_directory is not None:
args.replay_directory.mkdir(exist_ok=False)
for index, path in enumerate((args.row0, args.row1)):
audit = audit_module.TrialAudit(protocol, index)
seconds = defaultdict(float)
counts = defaultdict(int)
compressed = defaultdict(int)
chunks = []
chunk_size = 0
digest = hashlib.sha256()
profiler = cProfile.Profile()
def timed(name, fn, *values, **kwargs):
start = time.perf_counter()
value = fn(*values, **kwargs)
seconds[name] += time.perf_counter() - start
return value
def flush():
nonlocal chunks, chunk_size
if not chunks:
return
raw = b''.join(chunks)
for name, codec in [('gzip_chunk_level1', lambda b: gzip.compress(b, compresslevel=1, mtime=0)),
('xz_chunk_preset0', lambda b: lzma.compress(b, preset=0))]:
data = timed(name, codec, raw)
compressed[name] += len(data)
assert (gzip.decompress(data) if name.startswith('gzip') else lzma.decompress(data)) == raw
counts['chunks'] += 1
chunks, chunk_size = [], 0
start = time.perf_counter()
with gzip.open(path, 'rb') as source:
while True:
raw = timed('gzip_readline', source.readline, audit_module.MAX_RECORD_BYTES + 1)
if not raw:
break
if len(raw) > audit_module.MAX_RECORD_BYTES or time.monotonic() >= deadline:
raise RuntimeError('offline benchmark bound')
record = timed('json_loads', json.loads, raw)
counts[record['kind']] += 1
safe = timed('json_safe', recording._json_safe, record)
encoded = timed('json_dumps_encode', lambda: (json.dumps(safe, allow_nan=False,
separators=(',', ':')) + '\n').encode())
assert encoded == raw # Complete serialized fields and order, not just parsed equality.
digest.update(encoded)
compressed['gzip_record_level1'] += len(timed('gzip_record_level1',
gzip.compress, encoded, compresslevel=1, mtime=0))
timed('plain_copy', audit_module.plain, record)
timed('recorder_copied', recording.copied, record)
causes = timed('audit_consume', audit.consume, record)
# Profile a separate prefix audit below; never alter this audit's ordering.
if len(chunks) == 64 or chunk_size + len(raw) > 8 * 1024 * 1024:
flush()
chunks.append(raw)
chunk_size += len(raw)
if causes or record['kind'] in ('initial', 'seam', 'step_result', 'terminal', 'capture_fault'):
flush()
# Real read-only aggregate walk, same frequency as original recording callback.
timed('artifact_size_scan', budget_module.artifact_size, args.budget_root, exclude_native=True)
flush()
elapsed = time.perf_counter() - start
assert audit.terminal is not None
report = audit.report()
original = json.loads((path.parent / 'metrics.json').read_text())
assert all(original[k] == value for k, value in report.items()), 'saved audit mismatch'
profile_audit = audit_module.TrialAudit(protocol, index)
with gzip.open(path, 'rb') as source:
for _ in range(161):
record = json.loads(source.readline())
profiler.runcall(profile_audit.consume, record)
stats = pstats.Stats(profiler)
top = sorted(stats.stats.items(), key=lambda item: item[1][3], reverse=True)[:20]
result['rows'].append({'path': str(path), 'original_bytes': path.stat().st_size,
'records': dict(counts), 'component_seconds': dict(seconds), 'measured_loop_seconds': elapsed,
'compressed_bytes': dict(compressed), 'serialized_sha256': digest.hexdigest(),
'saved_metrics_audit_equal': True,
'bytes_per_tick': {k: v / counts['tick'] for k, v in compressed.items()},
'projected_90000_tick_bytes': {k: 90000 * v / counts['tick'] for k, v in compressed.items()},
'audit_ticks_per_second': counts['tick'] / seconds['audit_consume'],
'audit_profile_prefix_161_records': [{'function': str(key), 'primitive_calls': value[0],
'calls': value[1], 'self_seconds': value[2], 'cumulative_seconds': value[3]} for key, value in top]})
if args.replay_directory is not None:
checks = defaultdict(int)
scan_seconds = 0.
def budget(stage, pending, written):
nonlocal scan_seconds
started = time.perf_counter()
size = (budget_module.artifact_size(args.budget_root, exclude_native=True)
+ budget_module.artifact_size(args.replay_directory))
scan_seconds += time.perf_counter() - started
checks[stage] += 1
if size + pending > 1024 ** 3 or time.monotonic() > deadline:
raise RuntimeError('offline replay budget')
started = time.perf_counter()
recorder = recording.EvaluationRecorder(protocol, index, args.replay_directory / f'row{index}',
budget_check=budget, supplemental=lambda _: {})
with gzip.open(path, 'rb') as source:
for line in source:
record = json.loads(line)
if record['kind'] == 'terminal':
stop_type = dataclasses.make_dataclass('SavedStop', [(k, object) for k in record['runtime_stop']])
acknowledged = []
live = recorder.finalize(stop=stop_type(**record['runtime_stop']),
external_failure=record['external_failure'], acknowledge=acknowledged.append)
assert len(acknowledged) == 1
else:
recorder._record(record)
replay_seconds = time.perf_counter() - started
assert not recorder.faults
assert all(live[k] == value for k, value in report.items())
# Production bounded reader, complete traversal including trailers/EOF.
spec = importlib.util.spec_from_file_location('_offline_reader', ROOT / 'scripts/audit_cycle_sim_evaluation.py')
reader = importlib.util.module_from_spec(spec)
spec.loader.exec_module(reader)
trace = recorder.directory / audit_module.TRACE_FILENAME
replay_audit = audit_module.TrialAudit(protocol, index)
actual_digest = hashlib.sha256()
decoded_records = 0
for line in reader.trace_lines(trace, compressed=True, max_bytes=128 * 1024 * 1024,
max_record_bytes=audit_module.MAX_RECORD_BYTES, deadline=deadline):
actual_digest.update(line)
decoded_records += 1
replay_audit.consume(json.loads(line))
assert actual_digest.hexdigest() == digest.hexdigest()
assert decoded_records == sum(v for k, v in counts.items() if k != 'chunks')
assert replay_audit.report() == report
result['rows'][-1]['production_replay'] = {
'trace': str(trace), 'trace_bytes': trace.stat().st_size,
'bytes_per_tick': trace.stat().st_size / counts['tick'],
'cpu_seconds_including_read_json_budget_fsync': replay_seconds,
'cpu_ticks_per_second': counts['tick'] / replay_seconds,
'budget_checks': dict(checks), 'budget_scan_seconds': scan_seconds,
'decoded_bytes_sha256_equal': True, 'decoded_records': decoded_records,
'audit_report_equal': True, 'saved_metrics_equal': True}
with args.output.open('x') as output:
json.dump(result, output, indent=2)
output.write('\n')
print(json.dumps(result, indent=2))
if __name__ == '__main__':
main()