"""Observability only: keep GPUOwnership decisions and ordering unchanged. Never waive an unknown client, retry a failed check, sleep for NVML, signal a process, or infer that a historical failure was an exit race. Exact query and owned-discovery observations are recorded before re-raising the original error. """ from collections import deque import traceback import rootxy_launch_v2 as base class ObservedGPUOwnership(base.GPUOwnership): def __init__(self, identity, emit): self.emit = emit self.recent = deque(maxlen=16) self._sample = {'phase': 'admission', 'start_boottime': base.b.boottime()} self.next_publication = 0 try: super().__init__(identity) except BaseException: self._finish(traceback.format_exc()) raise else: self._finish(None, force=True) def clients(self): clients = super().clients() if self._sample is not None: self._sample['nvml_clients'] = {k: sorted(v) for k, v in clients.items()} self._sample['nvml_query_finished_boottime'] = base.b.boottime() return clients def _finish(self, error, *, force=False): sample = self._sample self._sample = None sample['end_boottime'] = base.b.boottime() sample['error'] = error if error is not None: # These are explicitly AFTER-check facts, not an atomic NVML/proc view. try: table = base.owner.process_table() pids = {p for values in sample.get('nvml_clients', {}).values() for p in values} pids |= {p for p, _ in sample.get('owned_known', [])} sample['proc_after_failure'] = {str(p): table.get(p) for p in sorted(pids)} except BaseException: sample['proc_after_failure_error'] = traceback.format_exc() self.recent.append(sample) if force or error is not None or sample['end_boottime'] >= self.next_publication: self.next_publication = sample['end_boottime'] + 1 self.emit({'scope': 'Unchanged fail-closed decisions; non-atomic observations, no tolerance/retry.', 'recent_checks': list(self.recent)}) def check(self, owned, *, final=False): self._sample = {'phase': 'final' if final else 'live', 'start_boottime': base.b.boottime(), 'owned_known_before': sorted([list(x) for x in owned.known])} wrapper = self class Observation: def discover(self): active = owned.discover() wrapper._sample['active_owned'] = {str(p): list(v) for p, v in active.items()} wrapper._sample['owned_known'] = sorted([list(x) for x in owned.known]) wrapper._sample['owned_discovery_finished_boottime'] = base.b.boottime() return active try: result = super().check(Observation(), final=final) except BaseException: self._finish(traceback.format_exc()) raise else: self._sample['result'] = result self._finish(None, force=final) return result