Files
cdsl-cad/backend/app/cad_agent/adapters/sqlite_repository.py
T

466 lines
28 KiB
Python

"""SQLite repository: the sole mutable state authority for protocol v3."""
from __future__ import annotations
from contextlib import contextmanager
import json
from pathlib import Path
import sqlite3
from threading import RLock
from typing import Any, Iterator
from app.cad_agent.domain.errors import ErrorCode
from app.cad_agent.domain.state import PendingAction, TaskPhase, TaskState
from app.cad_agent.ports import InvocationRecord
class SqliteTaskRepository:
def __init__(self, database_path: Path) -> None:
self.database_path = database_path
self.database_path.parent.mkdir(parents=True, exist_ok=True)
self._lock = RLock()
self.protocol_reset = False
self._initialize()
@contextmanager
def _connection(self) -> Iterator[sqlite3.Connection]:
connection = sqlite3.connect(self.database_path, timeout=20, isolation_level=None)
try:
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON")
connection.execute("PRAGMA journal_mode = WAL")
yield connection
finally:
connection.close()
def _initialize(self) -> None:
with self._lock, self._connection() as connection:
# Protocol 3.2 intentionally has no migration path from the
# structured-only / review-loop task model. Deployment starts with
# an empty task database, as those tasks do not have immutable
# Markdown source artifacts to compile from.
existing = connection.execute("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'tasks'").fetchone()
if existing is not None and "'3.2'" not in str(existing[0] or ""):
self.protocol_reset = True
connection.executescript("""
DROP TABLE IF EXISTS outbox;
DROP TABLE IF EXISTS tool_audits;
DROP TABLE IF EXISTS usage_records;
DROP TABLE IF EXISTS invocations;
DROP TABLE IF EXISTS ledger;
DROP TABLE IF EXISTS model_capabilities;
DROP TABLE IF EXISTS tasks;
""")
connection.executescript(
"""
CREATE TABLE IF NOT EXISTS tasks (
task_id TEXT PRIMARY KEY,
protocol_version TEXT NOT NULL CHECK(protocol_version = '3.2'),
request TEXT NOT NULL,
phase TEXT NOT NULL,
state_version INTEGER NOT NULL,
active_revision TEXT NOT NULL DEFAULT '',
pending_action_json TEXT,
candidate_id TEXT NOT NULL DEFAULT '',
candidate_stage_id TEXT NOT NULL DEFAULT '',
repair_required INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
retry_from_phase TEXT NOT NULL DEFAULT '',
requirements_spec_path TEXT NOT NULL DEFAULT '',
requirements_document_path TEXT NOT NULL DEFAULT '',
completion_target_path TEXT NOT NULL DEFAULT '',
modeling_plan_path TEXT NOT NULL DEFAULT '',
feature_plan_path TEXT NOT NULL DEFAULT '',
feature_plan_hash TEXT NOT NULL DEFAULT '',
feature_stage_id TEXT NOT NULL DEFAULT '',
clarification_path TEXT NOT NULL DEFAULT '',
requirements_contract_path TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS invocations (
invocation_id TEXT PRIMARY KEY,
task_id TEXT NOT NULL REFERENCES tasks(task_id),
idempotency_key TEXT NOT NULL,
status TEXT NOT NULL,
result_json TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(task_id, idempotency_key)
);
CREATE TABLE IF NOT EXISTS ledger (
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
task_id TEXT NOT NULL REFERENCES tasks(task_id),
event_json TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS outbox (
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id TEXT NOT NULL REFERENCES tasks(task_id),
event_json TEXT NOT NULL,
published_at TEXT
);
CREATE TABLE IF NOT EXISTS usage_records (
usage_id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id TEXT NOT NULL REFERENCES tasks(task_id),
usage_json TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS tool_audits (
audit_id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id TEXT NOT NULL REFERENCES tasks(task_id),
audit_json TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS model_capabilities (
provider_id TEXT NOT NULL,
model_id TEXT NOT NULL,
schema_hash TEXT NOT NULL,
supported INTEGER NOT NULL,
report_json TEXT NOT NULL,
checked_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY(provider_id, model_id, schema_hash)
);
"""
)
def create_task(self, task_id: str, request: str) -> TaskState:
with self._lock, self._connection() as connection:
connection.execute(
"INSERT OR IGNORE INTO tasks(task_id, protocol_version, request, phase, state_version) VALUES (?, '3.2', ?, ?, 0)",
(task_id, request, TaskPhase.DRAFTING_REQUIREMENTS_DOCUMENT.value),
)
state = self.get_state(task_id)
if state is None:
raise RuntimeError("SQLite task insert was not visible")
return state
def get_state(self, task_id: str) -> TaskState | None:
with self._lock, self._connection() as connection:
row = connection.execute("SELECT * FROM tasks WHERE task_id = ?", (task_id,)).fetchone()
return self._state(row) if row is not None else None
def get_task_projection(self, task_id: str) -> dict[str, Any] | None:
state = self.get_state(task_id)
if state is None:
return None
events = self.ledger_events(task_id)
revisions = [
{
"revision_id": item["revision_id"], "status": "success", "visibility": "final" if state.phase == TaskPhase.COMPLETED and item["revision_id"] == state.active_revision else "checkpoint",
"cdsl_path": f"revisions/{item['revision_id']}/model.cdsl.json", "step_path": f"revisions/{item['revision_id']}/model.step", "glb_path": "" if item.get("preview_unavailable") else f"revisions/{item['revision_id']}/model.glb", "report_path": f"revisions/{item['revision_id']}/rebuild-report.json", "candidate_review_path": item.get("review_path", ""),
}
for item in events
if item.get("event") in {"accepted", "feature_node_verified"} and isinstance(item.get("revision_id"), str)
]
frozen = next((item for item in reversed(events) if item.get("event") == "requirements_compiled"), {})
verification_warnings = [
str(item) for item in frozen.get("verification_warnings") or () if str(item)
] if isinstance(frozen, dict) else []
status_event = next((
item for item in reversed(events)
if item.get("event") in {
"requirements_waiting_for_user", "waiting_retry", "call_budget_exhausted",
"no_progress_limit",
"candidate_runtime_execution_failure", "candidate_recovery_runtime_execution_failure",
"failed_author_format", "runtime_contract_invalid",
"completed_best_effort",
}
), {}) if state.phase in {TaskPhase.COMPLETED, TaskPhase.FAILED, TaskPhase.WAITING_RETRY, TaskPhase.WAITING_FOR_USER} else {}
questions = [str(item) for item in status_event.get("questions") or () if str(item)] if isinstance(status_event, dict) else []
issues = [str(item) for item in status_event.get("issues") or () if str(item)] if isinstance(status_event, dict) else []
return {
"schema_version": "3.2",
"task_id": state.task_id,
"phase": state.phase.value,
"lifecycle": self._lifecycle(state.phase),
"state_version": state.version,
"active_revision": state.active_revision,
"current_revision": state.active_revision,
"published_revision": state.active_revision if state.phase == TaskPhase.COMPLETED else "",
"pending_action": self._pending_payload(state.pending_action),
"active_candidate_id": state.candidate_id,
"repair_required": state.repair_required,
"last_error": state.last_error.value if state.last_error else "",
"retry_from_phase": state.retry_from_phase.value if state.retry_from_phase else "",
"requirements_spec_path": state.requirements_spec_path,
"requirements_document_path": state.requirements_document_path,
"completion_target_path": state.completion_target_path,
"modeling_plan_path": state.modeling_plan_path,
"feature_plan_path": state.feature_plan_path,
"feature_plan_hash": state.feature_plan_hash,
"current_feature_node_id": state.pending_feature.node_id if state.pending_feature else "",
"pending_feature": self._pending_payload(state.pending_feature),
"feature_nodes": self._feature_node_projection(events),
"clarification_path": state.clarification_path,
"requirements_contract_path": state.requirements_contract_path,
"verification_status": (
"completed_with_risks" if state.phase == TaskPhase.COMPLETED and (verification_warnings or state.last_error == ErrorCode.BEST_EFFORT_COMPLETED)
else "verified" if state.phase == TaskPhase.COMPLETED
else "pending"
),
"verification_warnings": verification_warnings,
"message": str(status_event.get("message") or "") if isinstance(status_event, dict) else "",
"questions": questions,
"issues": issues,
"blocker_type": (
"requirements_ambiguity" if state.phase == TaskPhase.WAITING_FOR_USER
else str(status_event.get("event") or "") if isinstance(status_event, dict) else ""
),
"user_action_required": state.phase == TaskPhase.WAITING_FOR_USER and bool(questions),
"action_ledger_summary": events[-12:],
"revisions": revisions,
}
@staticmethod
def _feature_node_projection(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Return ledger-backed execution evidence; API adds immutable plan fields."""
nodes: dict[str, dict[str, Any]] = {}
for event in events:
node_id = str(event.get("node_id") or "")
if not node_id:
continue
item = nodes.setdefault(node_id, {"node_id": node_id, "status": "pending", "attempt": 0})
if event.get("event") == "feature_node_scheduled":
item.update({"status": "running", "atomic_id": str(event.get("atomic_id") or ""), "priority": event.get("priority"), "claim_ids": event.get("claim_ids") or [], "depends_on": event.get("depends_on") or []})
elif event.get("event") == "feature_node_failed":
item.update({"status": "failed" if event.get("terminal") else "pending", "attempt": int(event.get("attempt") or 0), "failure_class": str(event.get("failure_class") or ""), "error": str(event.get("message") or "")})
elif event.get("event") == "feature_node_verified":
item.update({"status": "done", "revision_id": str(event.get("revision_id") or ""), "feature_id": str(event.get("feature_id") or ""), "evidence": event.get("claim_results") or []})
elif event.get("event") == "feature_node_invalidated" and item.get("status") != "done":
item["status"] = "invalidated"
return list(nodes.values())
def ledger_events(self, task_id: str) -> list[dict[str, Any]]:
with self._lock, self._connection() as connection:
rows = connection.execute("SELECT sequence, event_json, created_at FROM ledger WHERE task_id = ? ORDER BY sequence", (task_id,)).fetchall()
return [{"sequence": row["sequence"], "at": row["created_at"], **json.loads(row["event_json"])} for row in rows]
def invocation_records(self, task_id: str) -> list[dict[str, Any]]:
with self._lock, self._connection() as connection:
rows = connection.execute(
"SELECT invocation_id, idempotency_key, status, result_json, created_at, updated_at FROM invocations WHERE task_id = ? ORDER BY created_at, invocation_id",
(task_id,),
).fetchall()
return [
{
"invocation_id": str(row["invocation_id"]), "idempotency_key": str(row["idempotency_key"]),
"status": str(row["status"]), "result": json.loads(row["result_json"]) if row["result_json"] else None,
"created_at": str(row["created_at"]), "updated_at": str(row["updated_at"]),
}
for row in rows
]
def compare_and_swap(
self,
state: TaskState,
*,
events: list[dict[str, Any]] = (),
invocation_id: str | None = None,
invocation_result: dict[str, Any] | None = None,
) -> bool:
if (invocation_id is None) != (invocation_result is None):
raise ValueError("Invocation completion requires both invocation_id and invocation_result")
previous_version = state.version - 1
if previous_version < 0:
raise ValueError("State version must advance exactly once")
with self._lock, self._connection() as connection:
connection.execute("BEGIN IMMEDIATE")
try:
cursor = connection.execute(
"""UPDATE tasks SET phase = ?, state_version = ?, active_revision = ?, pending_action_json = ?,
candidate_id = ?, candidate_stage_id = ?, repair_required = ?, last_error = ?, retry_from_phase = ?, requirements_spec_path = ?,
requirements_document_path = ?, completion_target_path = ?, modeling_plan_path = ?, feature_plan_path = ?, feature_plan_hash = ?, feature_stage_id = ?, clarification_path = ?, requirements_contract_path = ?, updated_at = CURRENT_TIMESTAMP
WHERE task_id = ? AND state_version = ?""",
(
state.phase.value, state.version, state.active_revision,
json.dumps(self._pending_payload(state.pending_action), ensure_ascii=True) if state.pending_action else None,
state.candidate_id, state.candidate_stage_id,
int(state.repair_required), state.last_error.value if state.last_error else None,
state.retry_from_phase.value if state.retry_from_phase else "", state.requirements_spec_path,
state.requirements_document_path, state.completion_target_path, state.modeling_plan_path,
state.feature_plan_path, state.feature_plan_hash, state.feature_stage_id,
state.clarification_path, state.requirements_contract_path,
state.task_id, previous_version,
),
)
if cursor.rowcount != 1:
connection.execute("ROLLBACK")
return False
for event in events:
encoded = json.dumps(event, ensure_ascii=True, sort_keys=True)
connection.execute("INSERT INTO ledger(task_id, event_json) VALUES (?, ?)", (state.task_id, encoded))
connection.execute("INSERT INTO outbox(task_id, event_json) VALUES (?, ?)", (state.task_id, encoded))
if invocation_id is not None and invocation_result is not None:
finished = connection.execute(
"""UPDATE invocations
SET status = 'finished', result_json = ?, updated_at = CURRENT_TIMESTAMP
WHERE invocation_id = ? AND task_id = ? AND status = 'processing'""",
(json.dumps(invocation_result, ensure_ascii=True), invocation_id, state.task_id),
)
if finished.rowcount != 1:
raise ValueError("Invocation is not an active record for this state transition")
connection.execute("COMMIT")
return True
except Exception:
connection.execute("ROLLBACK")
raise
def begin_invocation(self, task_id: str, invocation_id: str, idempotency_key: str) -> InvocationRecord:
with self._lock, self._connection() as connection:
connection.execute("BEGIN IMMEDIATE")
try:
existing = connection.execute("SELECT * FROM invocations WHERE task_id = ? AND idempotency_key = ?", (task_id, idempotency_key)).fetchone()
if existing is not None:
connection.execute("COMMIT")
return self._invocation(existing)
connection.execute("INSERT INTO invocations(invocation_id, task_id, idempotency_key, status) VALUES (?, ?, ?, 'processing')", (invocation_id, task_id, idempotency_key))
connection.execute("COMMIT")
return InvocationRecord(invocation_id, idempotency_key, "processing")
except Exception:
connection.execute("ROLLBACK")
raise
def get_invocation(self, task_id: str, invocation_id: str) -> InvocationRecord | None:
with self._lock, self._connection() as connection:
row = connection.execute(
"SELECT * FROM invocations WHERE task_id = ? AND invocation_id = ?",
(task_id, invocation_id),
).fetchone()
return self._invocation(row) if row is not None else None
def finish_invocation(self, invocation_id: str, result: dict[str, Any]) -> None:
with self._lock, self._connection() as connection:
row = connection.execute("SELECT status, result_json FROM invocations WHERE invocation_id = ?", (invocation_id,)).fetchone()
if row is None:
raise ValueError("Unknown invocation")
if str(row["status"]) == "finished":
# A winning concurrent command has already recorded the only
# durable result for this idempotency key. Never overwrite it.
return
cursor = connection.execute(
"UPDATE invocations SET status = 'finished', result_json = ?, updated_at = CURRENT_TIMESTAMP WHERE invocation_id = ? AND status = 'processing'",
(json.dumps(result, ensure_ascii=True), invocation_id),
)
if cursor.rowcount != 1:
raise ValueError("Invocation could not be completed")
def append_outbox(self, task_id: str, event: dict[str, Any]) -> None:
with self._lock, self._connection() as connection:
connection.execute("INSERT INTO outbox(task_id, event_json) VALUES (?, ?)", (task_id, json.dumps(event, ensure_ascii=True, sort_keys=True)))
def pending_outbox(self, limit: int = 100, *, task_id: str | None = None) -> list[dict[str, Any]]:
with self._lock, self._connection() as connection:
if task_id is None:
rows = connection.execute("SELECT event_id, task_id, event_json FROM outbox WHERE published_at IS NULL ORDER BY event_id LIMIT ?", (limit,)).fetchall()
else:
rows = connection.execute("SELECT event_id, task_id, event_json FROM outbox WHERE published_at IS NULL AND task_id = ? ORDER BY event_id LIMIT ?", (task_id, limit)).fetchall()
return [{"event_id": row["event_id"], "task_id": row["task_id"], **json.loads(row["event_json"])} for row in rows]
def mark_outbox_published(self, event_id: int) -> None:
with self._lock, self._connection() as connection:
connection.execute("UPDATE outbox SET published_at = CURRENT_TIMESTAMP WHERE event_id = ? AND published_at IS NULL", (event_id,))
def record_usage(self, task_id: str, payload: dict[str, Any]) -> None:
with self._lock, self._connection() as connection:
connection.execute("INSERT INTO usage_records(task_id, usage_json) VALUES (?, ?)", (task_id, json.dumps(payload, ensure_ascii=True, sort_keys=True)))
def record_tool_audit(self, task_id: str, payload: dict[str, Any]) -> None:
"""Persist a diagnostic structured-output audit separately from usage."""
with self._lock, self._connection() as connection:
connection.execute(
"INSERT INTO tool_audits(task_id, audit_json) VALUES (?, ?)",
(task_id, json.dumps(payload, ensure_ascii=True, sort_keys=True)),
)
def tool_audits(self, task_id: str) -> list[dict[str, Any]]:
with self._lock, self._connection() as connection:
rows = connection.execute(
"SELECT audit_id, audit_json, created_at FROM tool_audits WHERE task_id = ? ORDER BY audit_id",
(task_id,),
).fetchall()
return [
{"audit_id": int(row["audit_id"]), "at": str(row["created_at"]), **json.loads(row["audit_json"])}
for row in rows
]
def usage_summary(self, task_id: str) -> dict[str, Any]:
with self._lock, self._connection() as connection:
rows = connection.execute("SELECT usage_json FROM usage_records WHERE task_id = ? ORDER BY usage_id", (task_id,)).fetchall()
values = [json.loads(row["usage_json"]) for row in rows]
return {
"calls": len(values),
"prompt_tokens": sum(int(item.get("prompt_tokens") or 0) for item in values if isinstance(item, dict)),
"completion_tokens": sum(int(item.get("completion_tokens") or 0) for item in values if isinstance(item, dict)),
"context_chars": sum(int(item.get("context_chars") or 0) for item in values if isinstance(item, dict)),
"records": values,
}
def running_task_ids(self) -> list[str]:
phases = tuple(phase.value for phase in TaskPhase if phase not in {TaskPhase.COMPLETED, TaskPhase.FAILED, TaskPhase.CANCELLED, TaskPhase.WAITING_FOR_USER, TaskPhase.WAITING_RETRY})
placeholders = ", ".join("?" for _ in phases)
with self._lock, self._connection() as connection:
rows = connection.execute(f"SELECT task_id FROM tasks WHERE phase IN ({placeholders}) ORDER BY created_at", phases).fetchall()
return [str(row["task_id"]) for row in rows]
def model_capability(self, provider_id: str, model_id: str, schema_hash: str) -> dict[str, Any] | None:
with self._lock, self._connection() as connection:
row = connection.execute("SELECT supported, report_json, checked_at FROM model_capabilities WHERE provider_id = ? AND model_id = ? AND schema_hash = ?", (provider_id, model_id, schema_hash)).fetchone()
return {"supported": bool(row["supported"]), "report": json.loads(row["report_json"]), "checked_at": row["checked_at"]} if row else None
def record_model_capability(self, provider_id: str, model_id: str, schema_hash: str, report: dict[str, Any]) -> None:
with self._lock, self._connection() as connection:
connection.execute("""INSERT INTO model_capabilities(provider_id, model_id, schema_hash, supported, report_json)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(provider_id, model_id, schema_hash) DO UPDATE SET supported = excluded.supported, report_json = excluded.report_json, checked_at = CURRENT_TIMESTAMP""", (provider_id, model_id, schema_hash, int(bool(report.get("supported"))), json.dumps(report, ensure_ascii=True, sort_keys=True)))
@staticmethod
def _state(row: sqlite3.Row) -> TaskState:
raw_pending = json.loads(row["pending_action_json"]) if row["pending_action_json"] else None
pending = PendingAction(
action_id=raw_pending["action_id"], working_head=raw_pending["working_head"], intent=raw_pending["intent"],
requirement_ids=tuple(raw_pending["requirement_ids"]), atomic_id=raw_pending["atomic_id"],
expected_change=raw_pending["expected_change"], contract_hash=raw_pending["contract_hash"], idempotency_key=raw_pending["idempotency_key"],
node_id=str(raw_pending.get("node_id") or ""), plan_hash=str(raw_pending.get("plan_hash") or ""),
claim_ids=tuple(raw_pending.get("claim_ids") or ()), depends_on_node_ids=tuple(raw_pending.get("depends_on_node_ids") or ()),
) if isinstance(raw_pending, dict) else None
return TaskState(
task_id=str(row["task_id"]), phase=TaskPhase(str(row["phase"])), version=int(row["state_version"]),
active_revision=str(row["active_revision"] or ""), pending_action=pending,
candidate_id=str(row["candidate_id"] or ""), candidate_stage_id=str(row["candidate_stage_id"] or ""),
repair_required=bool(row["repair_required"]),
last_error=ErrorCode(str(row["last_error"])) if row["last_error"] else None,
retry_from_phase=TaskPhase(str(row["retry_from_phase"])) if row["retry_from_phase"] else None,
requirements_spec_path=str(row["requirements_spec_path"] or ""),
requirements_document_path=str(row["requirements_document_path"] or ""),
completion_target_path=str(row["completion_target_path"] or ""),
modeling_plan_path=str(row["modeling_plan_path"] or ""),
feature_plan_path=str(row["feature_plan_path"] or ""),
feature_plan_hash=str(row["feature_plan_hash"] or ""),
feature_stage_id=str(row["feature_stage_id"] or ""),
clarification_path=str(row["clarification_path"] or ""),
requirements_contract_path=str(row["requirements_contract_path"] or ""),
)
@staticmethod
def _pending_payload(pending: PendingAction | None) -> dict[str, Any] | None:
if pending is None:
return None
return {"action_id": pending.action_id, "working_head": pending.working_head, "intent": pending.intent, "requirement_ids": list(pending.requirement_ids), "atomic_id": pending.atomic_id, "expected_change": pending.expected_change, "contract_hash": pending.contract_hash, "idempotency_key": pending.idempotency_key, "node_id": pending.node_id, "plan_hash": pending.plan_hash, "claim_ids": list(pending.claim_ids), "depends_on_node_ids": list(pending.depends_on_node_ids)}
@staticmethod
def _invocation(row: sqlite3.Row) -> InvocationRecord:
return InvocationRecord(str(row["invocation_id"]), str(row["idempotency_key"]), str(row["status"]), json.loads(row["result_json"]) if row["result_json"] else None)
@staticmethod
def _lifecycle(phase: TaskPhase) -> str:
if phase == TaskPhase.COMPLETED:
return "completed"
if phase == TaskPhase.FAILED:
return "failed"
if phase == TaskPhase.CANCELLED:
return "cancelled"
if phase in {TaskPhase.WAITING_RETRY, TaskPhase.WAITING_FOR_USER}:
return phase.value.lower()
return "running"