"""Operator-invoked production recovery check; creates no model credentials or calls.""" import http.client import json import subprocess import time HOST = "cadworld-sim.robotquan.com" CONTAINER = "cadworld-sim-decision-1" def request(path, method="GET", headers=None, body=None): connection = http.client.HTTPSConnection(HOST, timeout=10) connection.request(method, path, body=body, headers=headers or {}) response = connection.getresponse() status, cookie = response.status, response.getheader("Set-Cookie") response.read() connection.close() return status, cookie def restart_count(): return int( subprocess.check_output( ["docker", "inspect", CONTAINER, "--format", "{{.RestartCount}}"], universal_newlines=True, ) ) def main(): status, cookie = request( "/api/decision/v1/session", "POST", {"Origin": "https://" + HOST, "Content-Type": "application/json"}, "{}", ) if status != 200 or not cookie: raise RuntimeError("session bootstrap failed") cookie = cookie.split(";", 1)[0] # Never print or persist the credential. before = restart_count() # Kill only this application's Python worker, not Docker/1Panel/host PID 1. code = """import os,signal for ident in os.listdir('/proc'): if ident.isdigit(): try: raw=open('/proc/'+ident+'/cmdline','rb').read() except OSError: continue if raw.startswith(b'python\\x00-m\\x00decision_server\\x00'): os.kill(int(ident),signal.SIGKILL) break else: raise RuntimeError('worker not found') """ # Tini may terminate the exec probe together with the killed worker (137). probe = subprocess.run(["docker", "exec", CONTAINER, "python", "-c", code], check=False) if probe.returncode not in (0, 137): raise RuntimeError("worker crash probe failed") deadline = time.monotonic() + 60 while time.monotonic() < deadline: time.sleep(1) try: status, _ = request("/api/decision/v1/status", headers={"Cookie": cookie}) if status == 401 and restart_count() > before: print( json.dumps( { "automaticRestart": True, "oldSessionStatus": status, "restartCount": restart_count(), "modelCalls": 0, } ) ) return except OSError: pass raise RuntimeError("automatic recovery not verified") if __name__ == "__main__": main()