#!/usr/bin/env python3 """CPU-only saved-record component replay; no simulator, inference or training. Private namespace packages bypass task registration, for offline measurement only, never for identity verification. Originals are read-only; outputs are exclusive. """ import argparse import cProfile import dataclasses import gzip import hashlib import importlib.util import json import lzma import pstats import sys import time import types from collections import defaultdict from pathlib import Path ROOT = Path(__file__).resolve().parents[1] def modules(): base = ROOT / 'source/SkillFactory/SkillFactory' for relative in ('', '.legs', '.legs.contact_cycle', '.legs.contact_cycle_sim'): name = 'SkillFactory' + relative package = types.ModuleType(name) package.__path__ = [str(base / relative.lstrip('.').replace('.', '/'))] sys.modules[name] = package from SkillFactory.legs.contact_cycle_sim import evaluation_audit, evaluation_budget, evaluation_protocol path = base / 'tasks/manager_based/contact_cycle_sim/evaluation_recording.py' spec = importlib.util.spec_from_file_location('_offline_recording', path) recording = importlib.util.module_from_spec(spec) spec.loader.exec_module(recording) return evaluation_protocol, evaluation_audit, evaluation_budget, recording def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--row0', type=Path, required=True) parser.add_argument('--row1', type=Path, required=True) parser.add_argument('--budget-root', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--replay-directory', type=Path) args = parser.parse_args() protocol_module, audit_module, budget_module, recording = modules() protocol = protocol_module.load_protocol(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json') result = {'scope': 'CPU saved-record replay only; not end-to-end or GPU throughput', 'python': sys.version, 'rows': [], 'budget_root': str(args.budget_root)} deadline = time.monotonic() + 1800 if args.replay_directory is not None: args.replay_directory.mkdir(exist_ok=False) for index, path in enumerate((args.row0, args.row1)): audit = audit_module.TrialAudit(protocol, index) seconds = defaultdict(float) counts = defaultdict(int) compressed = defaultdict(int) chunks = [] chunk_size = 0 digest = hashlib.sha256() profiler = cProfile.Profile() def timed(name, fn, *values, **kwargs): start = time.perf_counter() value = fn(*values, **kwargs) seconds[name] += time.perf_counter() - start return value def flush(): nonlocal chunks, chunk_size if not chunks: return raw = b''.join(chunks) for name, codec in [('gzip_chunk_level1', lambda b: gzip.compress(b, compresslevel=1, mtime=0)), ('xz_chunk_preset0', lambda b: lzma.compress(b, preset=0))]: data = timed(name, codec, raw) compressed[name] += len(data) assert (gzip.decompress(data) if name.startswith('gzip') else lzma.decompress(data)) == raw counts['chunks'] += 1 chunks, chunk_size = [], 0 start = time.perf_counter() with gzip.open(path, 'rb') as source: while True: raw = timed('gzip_readline', source.readline, audit_module.MAX_RECORD_BYTES + 1) if not raw: break if len(raw) > audit_module.MAX_RECORD_BYTES or time.monotonic() >= deadline: raise RuntimeError('offline benchmark bound') record = timed('json_loads', json.loads, raw) counts[record['kind']] += 1 safe = timed('json_safe', recording._json_safe, record) encoded = timed('json_dumps_encode', lambda: (json.dumps(safe, allow_nan=False, separators=(',', ':')) + '\n').encode()) assert encoded == raw # Complete serialized fields and order, not just parsed equality. digest.update(encoded) compressed['gzip_record_level1'] += len(timed('gzip_record_level1', gzip.compress, encoded, compresslevel=1, mtime=0)) timed('plain_copy', audit_module.plain, record) timed('recorder_copied', recording.copied, record) causes = timed('audit_consume', audit.consume, record) # Profile a separate prefix audit below; never alter this audit's ordering. if len(chunks) == 64 or chunk_size + len(raw) > 8 * 1024 * 1024: flush() chunks.append(raw) chunk_size += len(raw) if causes or record['kind'] in ('initial', 'seam', 'step_result', 'terminal', 'capture_fault'): flush() # Real read-only aggregate walk, same frequency as original recording callback. timed('artifact_size_scan', budget_module.artifact_size, args.budget_root, exclude_native=True) flush() elapsed = time.perf_counter() - start assert audit.terminal is not None report = audit.report() original = json.loads((path.parent / 'metrics.json').read_text()) assert all(original[k] == value for k, value in report.items()), 'saved audit mismatch' profile_audit = audit_module.TrialAudit(protocol, index) with gzip.open(path, 'rb') as source: for _ in range(161): record = json.loads(source.readline()) profiler.runcall(profile_audit.consume, record) stats = pstats.Stats(profiler) top = sorted(stats.stats.items(), key=lambda item: item[1][3], reverse=True)[:20] result['rows'].append({'path': str(path), 'original_bytes': path.stat().st_size, 'records': dict(counts), 'component_seconds': dict(seconds), 'measured_loop_seconds': elapsed, 'compressed_bytes': dict(compressed), 'serialized_sha256': digest.hexdigest(), 'saved_metrics_audit_equal': True, 'bytes_per_tick': {k: v / counts['tick'] for k, v in compressed.items()}, 'projected_90000_tick_bytes': {k: 90000 * v / counts['tick'] for k, v in compressed.items()}, 'audit_ticks_per_second': counts['tick'] / seconds['audit_consume'], 'audit_profile_prefix_161_records': [{'function': str(key), 'primitive_calls': value[0], 'calls': value[1], 'self_seconds': value[2], 'cumulative_seconds': value[3]} for key, value in top]}) if args.replay_directory is not None: checks = defaultdict(int) scan_seconds = 0. def budget(stage, pending, written): nonlocal scan_seconds started = time.perf_counter() size = (budget_module.artifact_size(args.budget_root, exclude_native=True) + budget_module.artifact_size(args.replay_directory)) scan_seconds += time.perf_counter() - started checks[stage] += 1 if size + pending > 1024 ** 3 or time.monotonic() > deadline: raise RuntimeError('offline replay budget') started = time.perf_counter() recorder = recording.EvaluationRecorder(protocol, index, args.replay_directory / f'row{index}', budget_check=budget, supplemental=lambda _: {}) with gzip.open(path, 'rb') as source: for line in source: record = json.loads(line) if record['kind'] == 'terminal': stop_type = dataclasses.make_dataclass('SavedStop', [(k, object) for k in record['runtime_stop']]) acknowledged = [] live = recorder.finalize(stop=stop_type(**record['runtime_stop']), external_failure=record['external_failure'], acknowledge=acknowledged.append) assert len(acknowledged) == 1 else: recorder._record(record) replay_seconds = time.perf_counter() - started assert not recorder.faults assert all(live[k] == value for k, value in report.items()) # Production bounded reader, complete traversal including trailers/EOF. spec = importlib.util.spec_from_file_location('_offline_reader', ROOT / 'scripts/audit_cycle_sim_evaluation.py') reader = importlib.util.module_from_spec(spec) spec.loader.exec_module(reader) trace = recorder.directory / audit_module.TRACE_FILENAME replay_audit = audit_module.TrialAudit(protocol, index) actual_digest = hashlib.sha256() decoded_records = 0 for line in reader.trace_lines(trace, compressed=True, max_bytes=128 * 1024 * 1024, max_record_bytes=audit_module.MAX_RECORD_BYTES, deadline=deadline): actual_digest.update(line) decoded_records += 1 replay_audit.consume(json.loads(line)) assert actual_digest.hexdigest() == digest.hexdigest() assert decoded_records == sum(v for k, v in counts.items() if k != 'chunks') assert replay_audit.report() == report result['rows'][-1]['production_replay'] = { 'trace': str(trace), 'trace_bytes': trace.stat().st_size, 'bytes_per_tick': trace.stat().st_size / counts['tick'], 'cpu_seconds_including_read_json_budget_fsync': replay_seconds, 'cpu_ticks_per_second': counts['tick'] / replay_seconds, 'budget_checks': dict(checks), 'budget_scan_seconds': scan_seconds, 'decoded_bytes_sha256_equal': True, 'decoded_records': decoded_records, 'audit_report_equal': True, 'saved_metrics_equal': True} with args.output.open('x') as output: json.dump(result, output, indent=2) output.write('\n') print(json.dumps(result, indent=2)) if __name__ == '__main__': main()