#!/usr/bin/env node /** * Build execution-validated STEP -> DesignIR trajectories. * * Agent A is an OpenAI-compatible language model. Agent B is the deterministic * CAD-kernel acceptance program. The teacher STEP is never included in Agent A * input and is used only by Agent B after isolated reconstruction. */ import fs from "node:fs"; import path from "node:path"; import process from "node:process"; import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; import { spawnSync } from "node:child_process"; const HERE = path.dirname(fileURLToPath(import.meta.url)); const PIPELINE = path.resolve(HERE, ".."); const REPO = path.resolve(PIPELINE, ".."); const requireFromStudio = createRequire( path.join(REPO, "cad-agent-studio", "package.json"), ); const YAML = requireFromStudio("yaml"); const CONFIG_PATH = path.join(REPO, "llm.config.yaml"); const EVIDENCE_DIR = path.join(PIPELINE, "runs", "evidence"); const TRAJECTORY_DIR = path.join(PIPELINE, "runs", "trajectory"); const SPLIT_PATH = path.join(TRAJECTORY_DIR, "splits.json"); const PYTHON = path.join(REPO, "text-to-cad", ".venv", "bin", "python"); const DESIGNIR_CLI = path.join(PIPELINE, "scripts", "designir_pipeline.py"); const EXAMPLE_PATH = path.join( PIPELINE, "examples", "raised_hub_flange.designir.json", ); function readJson(file) { return JSON.parse(fs.readFileSync(file, "utf8")); } function writeJson(file, value) { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); } function argsObject(argv) { const result = { _: [] }; for (let index = 0; index < argv.length; index += 1) { const value = argv[index]; if (!value.startsWith("--")) { result._.push(value); continue; } const key = value.slice(2); const next = argv[index + 1]; if (!next || next.startsWith("--")) { result[key] = true; } else { result[key] = next; index += 1; } } return result; } function evidenceFiles() { return fs .readdirSync(EVIDENCE_DIR) .filter((name) => name.endsWith(".case.json")) .map((name) => path.join(EVIDENCE_DIR, name)) .sort(); } function allocate(total, counts) { const population = Object.values(counts).reduce((sum, value) => sum + value, 0); const rows = Object.entries(counts).map(([family, count]) => { const exact = (count * total) / population; return { family, value: Math.floor(exact), remainder: exact % 1 }; }); let remaining = total - rows.reduce((sum, row) => sum + row.value, 0); rows.sort((left, right) => right.remainder - left.remainder); for (const row of rows) { if (remaining <= 0) break; row.value += 1; remaining -= 1; } return Object.fromEntries(rows.map((row) => [row.family, row.value])); } function initSplits() { const unique = new Map(); for (const file of evidenceFiles()) { const payload = readJson(file); const geometry = payload.geometry_evidence || {}; if (!geometry.reconstruction_eligible) continue; const id = String(payload.provenance?.source_sha256 || payload.case_id); if (unique.has(id)) continue; unique.set(id, { id, family: String(payload.design_ir?.part_family || "unknown"), evidence: file, teacher: String(payload.provenance?.source_path || ""), face_count: Number(geometry.face_count || 0), }); } const byFamily = {}; for (const item of unique.values()) { (byFamily[item.family] ||= []).push(item); } for (const items of Object.values(byFamily)) { items.sort((left, right) => left.id.localeCompare(right.id)); } const counts = Object.fromEntries( Object.entries(byFamily).map(([family, items]) => [family, items.length]), ); const testQuota = allocate(100, counts); const validationQuota = allocate(75, counts); const splits = { train: [], validation: [], test: [] }; for (const [family, items] of Object.entries(byFamily)) { const testCount = testQuota[family] || 0; const validationCount = validationQuota[family] || 0; splits.test.push(...items.slice(0, testCount)); splits.validation.push( ...items.slice(testCount, testCount + validationCount), ); splits.train.push(...items.slice(testCount + validationCount)); } for (const values of Object.values(splits)) { values.sort( (left, right) => left.face_count - right.face_count || left.id.localeCompare(right.id), ); } const payload = { schema_version: "1.0", split_kind: "sha256_isolated_stratified_trajectory_split", policy: { train_count: splits.train.length, validation_count: splits.validation.length, test_count: splits.test.length, teacher_step_visible_to_reconstruction_agent: false, heldout_cases_allowed_in_experience_induction: false, }, splits, }; writeJson(SPLIT_PATH, payload); return payload; } function loadProvider(requestedProvider, requestedModel) { const config = YAML.parse(fs.readFileSync(CONFIG_PATH, "utf8")); const providerId = requestedProvider || config.defaultProvider; const provider = config.providers[providerId]; if (!provider) throw new Error(`Unknown provider ${providerId}`); const apiKey = String(provider.apiKey || "").trim() || process.env[String(provider.apiKeyEnv || "")] || ""; if (!apiKey) throw new Error(`No API key configured for ${providerId}`); const defaultModel = String(config.defaultModel || "").split(":").pop(); const model = requestedModel || provider.models?.default || defaultModel; return { providerId, model, apiKey, baseURL: String(provider.baseURL).replace(/\/+$/, ""), }; } function compactEvidence(payload, maxSurfaces = 80) { const geometry = payload.geometry_evidence || {}; const design = payload.design_ir || {}; return { evidence_kind: "final_brep_observation_not_source_history", geometry: { solid_count: geometry.solid_count, face_count: geometry.face_count, volume: geometry.volume, bounding_box: geometry.bounding_box, analytic_surface_counts: geometry.analytic_surface_counts, analytic_surfaces: (geometry.analytic_surfaces || []).slice(0, maxSurfaces), }, inferred_hypotheses: { family: design.part_family, features: design.features, constraints: design.constraints, semantic_parameter_candidates: design.semantic_parameter_candidates, }, }; } function reconstructionPrompt(evidence, previous, feedback, experience = []) { const example = readJson(EXAMPLE_PATH); return [ "You are Agent A, a STEP reverse-engineering program synthesizer.", "Generate an independent executable DesignIR 2.0 JSON object from final-BRep evidence.", "The source STEP is unavailable. Never reference, import, embed, or describe it as a base feature.", "STEP does not contain source history. Use canonical reconstruction order and explicitly inferred semantic names.", "Current executable operations are exactly: extrude_circle, extrude_rectangle, add_cylinder, through_hole, polar_hole_pattern.", "Cylinder and extrusion primitives may use any observed axis. Hole cutters currently require the positive Z axis.", "Preserve the observed absolute coordinate frame and orientation.", "For cylinder and extrusion features, set anchor to geometric_center and center to the observed 3D bounding-box center unless there is strong evidence for another placement.", "The legacy/default anchor is base_center; never rely on that default for reverse engineering.", "A value reference object may contain exactly one key: parameter or expression. Do not add scale or arithmetic keys.", "Do not expose an editable parameter unless at least one feature directly references it. Non-driving measurements must be editable false or omitted.", "Use named editable parameters such as plate_thickness, hole_diameter, hole_count, bolt_circle_diameter, bore_diameter, overall_length, overall_width.", "Every editable parameter must be referenced by a feature.", "Every perturbation must contain parameter, scale, expected_change, and preserve.", "Use an empty list for constraints when none are justified.", "Return JSON only, without Markdown.", "", "REFERENCE FORMAT:", JSON.stringify(example), "", "PRIVATE GEOMETRY EVIDENCE:", JSON.stringify(evidence), experience.length ? `\nVALIDATED CANDIDATE METHODS (apply only when relevant):\n${JSON.stringify(experience)}` : "", previous ? `\nPREVIOUS CANDIDATE:\n${JSON.stringify(previous)}` : "", feedback ? `\nVALIDATOR FEEDBACK:\n${feedback}` : "", ] .filter(Boolean) .join("\n"); } async function callModel(provider, prompt) { const response = await fetch(`${provider.baseURL}/chat/completions`, { method: "POST", headers: { Authorization: `Bearer ${provider.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: provider.model, temperature: 0.2, messages: [ { role: "system", content: "Produce normalized, editable parametric CAD design states. Output strict JSON.", }, { role: "user", content: prompt }, ], response_format: { type: "json_object" }, }), signal: AbortSignal.timeout(90_000), }); if (!response.ok) { const detail = (await response.text()).slice(0, 600); throw new Error(`LLM HTTP ${response.status}: ${detail}`); } const payload = await response.json(); const content = payload.choices?.[0]?.message?.content; if (!content) throw new Error("LLM returned no message content"); const cleaned = String(content) .replace(/^```(?:json)?\s*/i, "") .replace(/\s*```$/, ""); return { designir: JSON.parse(cleaned), usage: payload.usage || {}, }; } function runPython(arguments_) { const result = spawnSync(PYTHON, [DESIGNIR_CLI, ...arguments_], { cwd: REPO, encoding: "utf8", timeout: 120_000, }); const output = `${result.stdout || ""}\n${result.stderr || ""}`.trim(); return { ok: result.status === 0, status: result.status, output }; } async function runOne(job, provider, maxAttempts, options = {}) { const shortId = job.id.slice(0, 16); const root = path.join( options.attemptsRoot || path.join(TRAJECTORY_DIR, "attempts"), shortId, ); fs.mkdirSync(root, { recursive: true }); const evidencePayload = readJson(job.evidence); const evidence = compactEvidence(evidencePayload); const unsupportedSurfaces = Object.entries( evidence.geometry.analytic_surface_counts || {}, ) .filter(([kind, count]) => count > 0 && !["plane", "cylinder"].includes(kind)) .map(([kind]) => kind); if (unsupportedSurfaces.length) { const trajectory = { schema_version: "1.0", trajectory_kind: "execution_validated_step_to_designir", case_id: job.id, family: job.family, provider: provider.providerId, model: provider.model, attempts: [], state: "unsupported_feature_vocabulary", missing_capabilities: unsupportedSurfaces.map( (kind) => `reconstruct_${kind}_surface`, ), }; writeJson(path.join(root, "trajectory.json"), trajectory); return trajectory; } let previous = null; let feedback = ""; const attempts = []; for (let index = 1; index <= maxAttempts; index += 1) { const started = Date.now(); try { const generated = await callModel( provider, reconstructionPrompt( evidence, previous, feedback, options.experience || [], ), ); const designir = generated.designir; designir.model_id ||= `trajectory_${shortId}_${index}`; designir.applied_experience_ids = (options.experience || []).map( (method) => method.id, ); const designirPath = path.join(root, `attempt-${index}.designir.json`); const rebuiltPath = path.join(root, `attempt-${index}.step`); const reportPath = path.join(root, `attempt-${index}.acceptance.json`); writeJson(designirPath, designir); const rebuild = runPython([ "isolated-rebuild", designirPath, "--output", rebuiltPath, "--backend", "build123d", ]); if (!rebuild.ok) { feedback = `Isolated rebuild failed: ${rebuild.output}`; attempts.push({ index, state: "compile_failed", duration_ms: Date.now() - started, usage: generated.usage, feedback, }); previous = designir; continue; } const acceptance = runPython([ "accept", "--teacher", job.teacher, "--designir", designirPath, "--rebuilt", rebuiltPath, "--report", reportPath, ]); const report = fs.existsSync(reportPath) ? readJson(reportPath) : null; const state = acceptance.ok && report?.verdict === "pass" ? "replay_validated" : "acceptance_failed"; attempts.push({ index, state, duration_ms: Date.now() - started, usage: generated.usage, report: reportPath, }); if (state === "replay_validated") { const trajectory = { schema_version: "1.0", trajectory_kind: "execution_validated_step_to_designir", case_id: job.id, split: job.split, family: job.family, provider: provider.providerId, model: provider.model, selected_attempt: index, designir: designirPath, rebuilt_step: rebuiltPath, acceptance_report: reportPath, attempts, state, }; writeJson(path.join(root, "trajectory.json"), trajectory); return trajectory; } feedback = `Acceptance failed: ${JSON.stringify({ threshold_checks: report?.threshold_checks, geometry: report?.geometry ? { relative_volume_error: report.geometry.relative_volume_error, bounding_box_axis_error_mm: report.geometry.bounding_box_axis_error_mm, symmetric_difference_ratio: report.geometry.symmetric_difference_ratio, teacher_facts: report.geometry.teacher, rebuilt_facts: report.geometry.rebuilt, } : null, editability: report?.editability, })}`; previous = designir; } catch (error) { feedback = error instanceof Error ? error.message : String(error); attempts.push({ index, state: "agent_failed", duration_ms: Date.now() - started, feedback, }); } } const trajectory = { schema_version: "1.0", trajectory_kind: "execution_validated_step_to_designir", case_id: job.id, family: job.family, provider: provider.providerId, model: provider.model, attempts, state: "quarantine", }; writeJson(path.join(root, "trajectory.json"), trajectory); return trajectory; } function summarize() { const root = path.join(TRAJECTORY_DIR, "attempts"); const rows = fs.existsSync(root) ? fs .readdirSync(root) .map((name) => path.join(root, name, "trajectory.json")) .filter((file) => fs.existsSync(file)) .map(readJson) : []; const states = {}; for (const row of rows) states[row.state] = (states[row.state] || 0) + 1; return { schema_version: "1.0", report_kind: "trajectory_run_summary", case_count: rows.length, states, validated_trajectory_count: states.replay_validated || 0, quarantined_count: states.quarantine || 0, rows: rows.map((row) => ({ case_id: row.case_id, family: row.family, state: row.state, attempt_count: row.attempts?.length || 0, })), }; } function distillValidatedTrajectories() { const root = path.join(TRAJECTORY_DIR, "attempts"); const trajectories = fs.existsSync(root) ? fs .readdirSync(root) .map((name) => path.join(root, name, "trajectory.json")) .filter((file) => fs.existsSync(file)) .map(readJson) .filter((row) => row.state === "replay_validated") : []; const groups = new Map(); for (const trajectory of trajectories) { const designir = readJson(trajectory.designir); const signature = designir.features .map((feature) => ["add_cylinder", "extrude_circle"].includes(feature.operation) ? "cylinder" : feature.operation, ) .sort() .join("+"); const key = `${designir.family}::${signature}`; const group = groups.get(key) || { family: designir.family, signature, cases: [], parameterNames: new Map(), anchors: new Map(), operations: new Set(), }; group.cases.push(trajectory.case_id); for (const name of Object.keys(designir.parameters || {})) { group.parameterNames.set(name, (group.parameterNames.get(name) || 0) + 1); } for (const feature of designir.features || []) { group.operations.add(feature.operation); const anchor = feature.anchor || "base_center"; group.anchors.set(anchor, (group.anchors.get(anchor) || 0) + 1); } groups.set(key, group); } const candidates = [...groups.values()].map((group) => { const simpleCylinder = group.family === "simple_cylinder" && group.signature === "cylinder"; return { id: `trajectory.${group.family}.${group.signature.replaceAll("+", "_")}`, kind: "reconstruction_grammar", promotion_state: "candidate", applicability: simpleCylinder ? { family: group.family, evidence_conditions: [ "one dominant axis", "one distinct coaxial radius", "low topological complexity", "solid volume approximately pi * radius^2 * axial_length", ], } : { family: group.family, operations: [...group.operations].sort(), }, method: { guidance: simpleCylinder ? "Interpret the single cylindrical surface as the outer body, not a bore. Build one cylinder from the observed outer radius, axial length, axis, and pose. Name the driving parameters outer_radius and overall_length. Preserve the absolute frame with an explicit center and anchor; do not expose duplicate width/thickness/inner-diameter parameters." : "Reconstruct from analytic dimensions and axis evidence; preserve the observed absolute frame with an explicit center and anchor; expose only parameters that directly drive executable features.", canonical_parameter_names: simpleCylinder ? ["outer_radius", "overall_length"] : [], rejected_parameter_hypotheses: simpleCylinder ? ["coaxial_inner_diameter", "duplicate_transverse_envelope_parameters"] : [], observed_parameter_names: Object.fromEntries( [...group.parameterNames.entries()].sort(), ), observed_anchor_modes: Object.fromEntries( [...group.anchors.entries()].sort(), ), }, validation: { independent_replay_validated_case_count: new Set(group.cases).size, source_hashes: [...new Set(group.cases)].sort(), edit_contract_pass_required: true, positive_heldout_ab_required: true, heldout_ab_status: "not_run", }, }; }); const payload = { schema_version: "1.0", library_kind: "execution_validated_trajectory_candidates", status: candidates.length ? "candidates_only" : "empty", policy: { router_consumable: false, automatic_publish_allowed: false, minimum_independent_cases: 3, positive_heldout_ab_required: true, }, validated_trajectory_count: trajectories.length, candidate_count: candidates.length, candidates, }; const output = path.join(TRAJECTORY_DIR, "candidate-experience.json"); writeJson(output, payload); return { output, payload }; } function sanitizedMethods(candidatePath) { if (!candidatePath) return []; return (readJson(candidatePath).candidates || []).map((candidate) => ({ id: candidate.id, kind: candidate.kind, applicability: candidate.applicability, method: candidate.method, })); } function benchmarkScore(rows) { const passed = rows.filter((row) => row.state === "replay_validated").length; return { case_count: rows.length, replay_validated_count: passed, pass_rate: rows.length ? passed / rows.length : 0, }; } async function runBenchmark(args) { const split = String(args.split || "validation"); if (!["validation", "test"].includes(split)) { throw new Error("Benchmark split must be validation or test"); } const limit = Math.max(1, Number(args.limit || 10)); const maxAttempts = Math.max(1, Number(args.attempts || 3)); const maxFaces = Math.max(1, Number(args["max-faces"] || 24)); const requestedFamily = String(args.family || "").trim(); const manifest = fs.existsSync(SPLIT_PATH) ? readJson(SPLIT_PATH) : initSplits(); const jobs = (manifest.splits?.[split] || []) .filter((job) => !requestedFamily || job.family === requestedFamily) .filter((job) => job.face_count <= maxFaces) .slice(0, limit); const provider = loadProvider(args.provider, args.model); const candidatePath = String( args.experience || path.join(TRAJECTORY_DIR, "candidate-experience.json"), ); const experience = sanitizedMethods(candidatePath); const benchmarkRoot = path.join( TRAJECTORY_DIR, "benchmarks", `${split}-${requestedFamily || "all"}`, ); const baseline = []; const treatment = []; for (const job of jobs) { baseline.push( await runOne({ ...job, split }, provider, maxAttempts, { attemptsRoot: path.join(benchmarkRoot, "baseline"), }), ); treatment.push( await runOne({ ...job, split }, provider, maxAttempts, { attemptsRoot: path.join(benchmarkRoot, "experience"), experience, }), ); } const baselineScore = benchmarkScore(baseline); const treatmentScore = benchmarkScore(treatment); const report = { schema_version: "1.0", report_kind: "heldout_trajectory_ab", split, family: requestedFamily || "all", provider: provider.providerId, model: provider.model, candidate_method_ids: experience.map((method) => method.id), baseline: baselineScore, experience: treatmentScore, pass_rate_delta: treatmentScore.pass_rate - baselineScore.pass_rate, positive: treatmentScore.pass_rate > baselineScore.pass_rate, promotion_eligible: jobs.length >= 10 && treatmentScore.pass_rate > baselineScore.pass_rate, case_results: jobs.map((job, index) => ({ case_id: job.id, baseline_state: baseline[index].state, experience_state: treatment[index].state, })), }; writeJson(path.join(benchmarkRoot, "report.json"), report); return report; } async function main() { const args = argsObject(process.argv.slice(2)); const command = args._[0]; if (command === "init") { const payload = initSplits(); process.stdout.write( `${JSON.stringify({ split_path: SPLIT_PATH, policy: payload.policy }, null, 2)}\n`, ); return; } if (command === "summary") { const report = summarize(); writeJson(path.join(TRAJECTORY_DIR, "summary.json"), report); process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); return; } if (command === "distill") { const result = distillValidatedTrajectories(); process.stdout.write( `${JSON.stringify( { output: result.output, validated_trajectory_count: result.payload.validated_trajectory_count, candidate_count: result.payload.candidate_count, router_consumable: result.payload.policy.router_consumable, }, null, 2, )}\n`, ); return; } if (command === "benchmark") { const report = await runBenchmark(args); process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); return; } if (command !== "run") { throw new Error( "Usage: trajectory_pipeline.mjs init|run|summary|distill|benchmark", ); } const split = String(args.split || "train"); const limit = Math.max(1, Number(args.limit || 5)); const maxAttempts = Math.max(1, Number(args.attempts || 3)); const maxFaces = Math.max(1, Number(args["max-faces"] || 24)); const requestedFamily = String(args.family || "").trim(); const rerun = Boolean(args.rerun); const manifest = fs.existsSync(SPLIT_PATH) ? readJson(SPLIT_PATH) : initSplits(); const jobs = (manifest.splits?.[split] || []) .filter((job) => !requestedFamily || job.family === requestedFamily) .filter((job) => job.face_count <= maxFaces) .filter( (job) => rerun || !fs.existsSync( path.join( TRAJECTORY_DIR, "attempts", job.id.slice(0, 16), "trajectory.json", ), ), ) .slice(0, limit); const provider = loadProvider(args.provider, args.model); const results = []; for (const job of jobs) { const result = await runOne({ ...job, split }, provider, maxAttempts); results.push({ case_id: job.id, family: job.family, face_count: job.face_count, state: result.state, }); process.stdout.write(`${JSON.stringify(results.at(-1))}\n`); } const report = summarize(); writeJson(path.join(TRAJECTORY_DIR, "summary.json"), report); process.stdout.write(`${JSON.stringify({ completed: results.length, report }, null, 2)}\n`); } main().catch((error) => { process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); process.exitCode = 1; });