9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
198 lines
10 KiB
Python
198 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""CPU-only saved-record component replay; no simulator, inference or training.
|
|
|
|
Private namespace packages bypass task registration, for offline measurement only,
|
|
never for identity verification. Originals are read-only; outputs are exclusive.
|
|
"""
|
|
import argparse
|
|
import cProfile
|
|
import dataclasses
|
|
import gzip
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import lzma
|
|
import pstats
|
|
import sys
|
|
import time
|
|
import types
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def modules():
|
|
base = ROOT / 'source/SkillFactory/SkillFactory'
|
|
for relative in ('', '.legs', '.legs.contact_cycle', '.legs.contact_cycle_sim'):
|
|
name = 'SkillFactory' + relative
|
|
package = types.ModuleType(name)
|
|
package.__path__ = [str(base / relative.lstrip('.').replace('.', '/'))]
|
|
sys.modules[name] = package
|
|
from SkillFactory.legs.contact_cycle_sim import evaluation_audit, evaluation_budget, evaluation_protocol
|
|
path = base / 'tasks/manager_based/contact_cycle_sim/evaluation_recording.py'
|
|
spec = importlib.util.spec_from_file_location('_offline_recording', path)
|
|
recording = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(recording)
|
|
return evaluation_protocol, evaluation_audit, evaluation_budget, recording
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--row0', type=Path, required=True)
|
|
parser.add_argument('--row1', type=Path, required=True)
|
|
parser.add_argument('--budget-root', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--replay-directory', type=Path)
|
|
args = parser.parse_args()
|
|
protocol_module, audit_module, budget_module, recording = modules()
|
|
protocol = protocol_module.load_protocol(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json')
|
|
result = {'scope': 'CPU saved-record replay only; not end-to-end or GPU throughput',
|
|
'python': sys.version, 'rows': [], 'budget_root': str(args.budget_root)}
|
|
deadline = time.monotonic() + 1800
|
|
if args.replay_directory is not None:
|
|
args.replay_directory.mkdir(exist_ok=False)
|
|
for index, path in enumerate((args.row0, args.row1)):
|
|
audit = audit_module.TrialAudit(protocol, index)
|
|
seconds = defaultdict(float)
|
|
counts = defaultdict(int)
|
|
compressed = defaultdict(int)
|
|
chunks = []
|
|
chunk_size = 0
|
|
digest = hashlib.sha256()
|
|
profiler = cProfile.Profile()
|
|
|
|
def timed(name, fn, *values, **kwargs):
|
|
start = time.perf_counter()
|
|
value = fn(*values, **kwargs)
|
|
seconds[name] += time.perf_counter() - start
|
|
return value
|
|
|
|
def flush():
|
|
nonlocal chunks, chunk_size
|
|
if not chunks:
|
|
return
|
|
raw = b''.join(chunks)
|
|
for name, codec in [('gzip_chunk_level1', lambda b: gzip.compress(b, compresslevel=1, mtime=0)),
|
|
('xz_chunk_preset0', lambda b: lzma.compress(b, preset=0))]:
|
|
data = timed(name, codec, raw)
|
|
compressed[name] += len(data)
|
|
assert (gzip.decompress(data) if name.startswith('gzip') else lzma.decompress(data)) == raw
|
|
counts['chunks'] += 1
|
|
chunks, chunk_size = [], 0
|
|
|
|
start = time.perf_counter()
|
|
with gzip.open(path, 'rb') as source:
|
|
while True:
|
|
raw = timed('gzip_readline', source.readline, audit_module.MAX_RECORD_BYTES + 1)
|
|
if not raw:
|
|
break
|
|
if len(raw) > audit_module.MAX_RECORD_BYTES or time.monotonic() >= deadline:
|
|
raise RuntimeError('offline benchmark bound')
|
|
record = timed('json_loads', json.loads, raw)
|
|
counts[record['kind']] += 1
|
|
safe = timed('json_safe', recording._json_safe, record)
|
|
encoded = timed('json_dumps_encode', lambda: (json.dumps(safe, allow_nan=False,
|
|
separators=(',', ':')) + '\n').encode())
|
|
assert encoded == raw # Complete serialized fields and order, not just parsed equality.
|
|
digest.update(encoded)
|
|
compressed['gzip_record_level1'] += len(timed('gzip_record_level1',
|
|
gzip.compress, encoded, compresslevel=1, mtime=0))
|
|
timed('plain_copy', audit_module.plain, record)
|
|
timed('recorder_copied', recording.copied, record)
|
|
causes = timed('audit_consume', audit.consume, record)
|
|
# Profile a separate prefix audit below; never alter this audit's ordering.
|
|
if len(chunks) == 64 or chunk_size + len(raw) > 8 * 1024 * 1024:
|
|
flush()
|
|
chunks.append(raw)
|
|
chunk_size += len(raw)
|
|
if causes or record['kind'] in ('initial', 'seam', 'step_result', 'terminal', 'capture_fault'):
|
|
flush()
|
|
# Real read-only aggregate walk, same frequency as original recording callback.
|
|
timed('artifact_size_scan', budget_module.artifact_size, args.budget_root, exclude_native=True)
|
|
flush()
|
|
elapsed = time.perf_counter() - start
|
|
assert audit.terminal is not None
|
|
report = audit.report()
|
|
original = json.loads((path.parent / 'metrics.json').read_text())
|
|
assert all(original[k] == value for k, value in report.items()), 'saved audit mismatch'
|
|
profile_audit = audit_module.TrialAudit(protocol, index)
|
|
with gzip.open(path, 'rb') as source:
|
|
for _ in range(161):
|
|
record = json.loads(source.readline())
|
|
profiler.runcall(profile_audit.consume, record)
|
|
stats = pstats.Stats(profiler)
|
|
top = sorted(stats.stats.items(), key=lambda item: item[1][3], reverse=True)[:20]
|
|
result['rows'].append({'path': str(path), 'original_bytes': path.stat().st_size,
|
|
'records': dict(counts), 'component_seconds': dict(seconds), 'measured_loop_seconds': elapsed,
|
|
'compressed_bytes': dict(compressed), 'serialized_sha256': digest.hexdigest(),
|
|
'saved_metrics_audit_equal': True,
|
|
'bytes_per_tick': {k: v / counts['tick'] for k, v in compressed.items()},
|
|
'projected_90000_tick_bytes': {k: 90000 * v / counts['tick'] for k, v in compressed.items()},
|
|
'audit_ticks_per_second': counts['tick'] / seconds['audit_consume'],
|
|
'audit_profile_prefix_161_records': [{'function': str(key), 'primitive_calls': value[0],
|
|
'calls': value[1], 'self_seconds': value[2], 'cumulative_seconds': value[3]} for key, value in top]})
|
|
if args.replay_directory is not None:
|
|
checks = defaultdict(int)
|
|
scan_seconds = 0.
|
|
|
|
def budget(stage, pending, written):
|
|
nonlocal scan_seconds
|
|
started = time.perf_counter()
|
|
size = (budget_module.artifact_size(args.budget_root, exclude_native=True)
|
|
+ budget_module.artifact_size(args.replay_directory))
|
|
scan_seconds += time.perf_counter() - started
|
|
checks[stage] += 1
|
|
if size + pending > 1024 ** 3 or time.monotonic() > deadline:
|
|
raise RuntimeError('offline replay budget')
|
|
|
|
started = time.perf_counter()
|
|
recorder = recording.EvaluationRecorder(protocol, index, args.replay_directory / f'row{index}',
|
|
budget_check=budget, supplemental=lambda _: {})
|
|
with gzip.open(path, 'rb') as source:
|
|
for line in source:
|
|
record = json.loads(line)
|
|
if record['kind'] == 'terminal':
|
|
stop_type = dataclasses.make_dataclass('SavedStop', [(k, object) for k in record['runtime_stop']])
|
|
acknowledged = []
|
|
live = recorder.finalize(stop=stop_type(**record['runtime_stop']),
|
|
external_failure=record['external_failure'], acknowledge=acknowledged.append)
|
|
assert len(acknowledged) == 1
|
|
else:
|
|
recorder._record(record)
|
|
replay_seconds = time.perf_counter() - started
|
|
assert not recorder.faults
|
|
assert all(live[k] == value for k, value in report.items())
|
|
# Production bounded reader, complete traversal including trailers/EOF.
|
|
spec = importlib.util.spec_from_file_location('_offline_reader', ROOT / 'scripts/audit_cycle_sim_evaluation.py')
|
|
reader = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(reader)
|
|
trace = recorder.directory / audit_module.TRACE_FILENAME
|
|
replay_audit = audit_module.TrialAudit(protocol, index)
|
|
actual_digest = hashlib.sha256()
|
|
decoded_records = 0
|
|
for line in reader.trace_lines(trace, compressed=True, max_bytes=128 * 1024 * 1024,
|
|
max_record_bytes=audit_module.MAX_RECORD_BYTES, deadline=deadline):
|
|
actual_digest.update(line)
|
|
decoded_records += 1
|
|
replay_audit.consume(json.loads(line))
|
|
assert actual_digest.hexdigest() == digest.hexdigest()
|
|
assert decoded_records == sum(v for k, v in counts.items() if k != 'chunks')
|
|
assert replay_audit.report() == report
|
|
result['rows'][-1]['production_replay'] = {
|
|
'trace': str(trace), 'trace_bytes': trace.stat().st_size,
|
|
'bytes_per_tick': trace.stat().st_size / counts['tick'],
|
|
'cpu_seconds_including_read_json_budget_fsync': replay_seconds,
|
|
'cpu_ticks_per_second': counts['tick'] / replay_seconds,
|
|
'budget_checks': dict(checks), 'budget_scan_seconds': scan_seconds,
|
|
'decoded_bytes_sha256_equal': True, 'decoded_records': decoded_records,
|
|
'audit_report_equal': True, 'saved_metrics_equal': True}
|
|
with args.output.open('x') as output:
|
|
json.dump(result, output, indent=2)
|
|
output.write('\n')
|
|
print(json.dumps(result, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|