Jerry
736 lines
25 KiB
JavaScript
736 lines
25 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Build execution-validated STEP -> DesignIR trajectories.
|
|
*
|
|
* Agent A is an OpenAI-compatible language model. Agent B is the deterministic
|
|
* CAD-kernel acceptance program. The teacher STEP is never included in Agent A
|
|
* input and is used only by Agent B after isolated reconstruction.
|
|
*/
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import process from "node:process";
|
|
import { createRequire } from "node:module";
|
|
import { fileURLToPath } from "node:url";
|
|
import { spawnSync } from "node:child_process";
|
|
|
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
const PIPELINE = path.resolve(HERE, "..");
|
|
const REPO = path.resolve(PIPELINE, "..");
|
|
const requireFromStudio = createRequire(
|
|
path.join(REPO, "cad-agent-studio", "package.json"),
|
|
);
|
|
const YAML = requireFromStudio("yaml");
|
|
const CONFIG_PATH = path.join(REPO, "llm.config.yaml");
|
|
const EVIDENCE_DIR = path.join(PIPELINE, "runs", "evidence");
|
|
const TRAJECTORY_DIR = path.join(PIPELINE, "runs", "trajectory");
|
|
const SPLIT_PATH = path.join(TRAJECTORY_DIR, "splits.json");
|
|
const PYTHON = path.join(REPO, "text-to-cad", ".venv", "bin", "python");
|
|
const DESIGNIR_CLI = path.join(PIPELINE, "scripts", "designir_pipeline.py");
|
|
const EXAMPLE_PATH = path.join(
|
|
PIPELINE,
|
|
"examples",
|
|
"raised_hub_flange.designir.json",
|
|
);
|
|
|
|
function readJson(file) {
|
|
return JSON.parse(fs.readFileSync(file, "utf8"));
|
|
}
|
|
|
|
function writeJson(file, value) {
|
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
|
|
}
|
|
|
|
function argsObject(argv) {
|
|
const result = { _: [] };
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const value = argv[index];
|
|
if (!value.startsWith("--")) {
|
|
result._.push(value);
|
|
continue;
|
|
}
|
|
const key = value.slice(2);
|
|
const next = argv[index + 1];
|
|
if (!next || next.startsWith("--")) {
|
|
result[key] = true;
|
|
} else {
|
|
result[key] = next;
|
|
index += 1;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function evidenceFiles() {
|
|
return fs
|
|
.readdirSync(EVIDENCE_DIR)
|
|
.filter((name) => name.endsWith(".case.json"))
|
|
.map((name) => path.join(EVIDENCE_DIR, name))
|
|
.sort();
|
|
}
|
|
|
|
function allocate(total, counts) {
|
|
const population = Object.values(counts).reduce((sum, value) => sum + value, 0);
|
|
const rows = Object.entries(counts).map(([family, count]) => {
|
|
const exact = (count * total) / population;
|
|
return { family, value: Math.floor(exact), remainder: exact % 1 };
|
|
});
|
|
let remaining = total - rows.reduce((sum, row) => sum + row.value, 0);
|
|
rows.sort((left, right) => right.remainder - left.remainder);
|
|
for (const row of rows) {
|
|
if (remaining <= 0) break;
|
|
row.value += 1;
|
|
remaining -= 1;
|
|
}
|
|
return Object.fromEntries(rows.map((row) => [row.family, row.value]));
|
|
}
|
|
|
|
function initSplits() {
|
|
const unique = new Map();
|
|
for (const file of evidenceFiles()) {
|
|
const payload = readJson(file);
|
|
const geometry = payload.geometry_evidence || {};
|
|
if (!geometry.reconstruction_eligible) continue;
|
|
const id = String(payload.provenance?.source_sha256 || payload.case_id);
|
|
if (unique.has(id)) continue;
|
|
unique.set(id, {
|
|
id,
|
|
family: String(payload.design_ir?.part_family || "unknown"),
|
|
evidence: file,
|
|
teacher: String(payload.provenance?.source_path || ""),
|
|
face_count: Number(geometry.face_count || 0),
|
|
});
|
|
}
|
|
const byFamily = {};
|
|
for (const item of unique.values()) {
|
|
(byFamily[item.family] ||= []).push(item);
|
|
}
|
|
for (const items of Object.values(byFamily)) {
|
|
items.sort((left, right) => left.id.localeCompare(right.id));
|
|
}
|
|
const counts = Object.fromEntries(
|
|
Object.entries(byFamily).map(([family, items]) => [family, items.length]),
|
|
);
|
|
const testQuota = allocate(100, counts);
|
|
const validationQuota = allocate(75, counts);
|
|
const splits = { train: [], validation: [], test: [] };
|
|
for (const [family, items] of Object.entries(byFamily)) {
|
|
const testCount = testQuota[family] || 0;
|
|
const validationCount = validationQuota[family] || 0;
|
|
splits.test.push(...items.slice(0, testCount));
|
|
splits.validation.push(
|
|
...items.slice(testCount, testCount + validationCount),
|
|
);
|
|
splits.train.push(...items.slice(testCount + validationCount));
|
|
}
|
|
for (const values of Object.values(splits)) {
|
|
values.sort(
|
|
(left, right) =>
|
|
left.face_count - right.face_count || left.id.localeCompare(right.id),
|
|
);
|
|
}
|
|
const payload = {
|
|
schema_version: "1.0",
|
|
split_kind: "sha256_isolated_stratified_trajectory_split",
|
|
policy: {
|
|
train_count: splits.train.length,
|
|
validation_count: splits.validation.length,
|
|
test_count: splits.test.length,
|
|
teacher_step_visible_to_reconstruction_agent: false,
|
|
heldout_cases_allowed_in_experience_induction: false,
|
|
},
|
|
splits,
|
|
};
|
|
writeJson(SPLIT_PATH, payload);
|
|
return payload;
|
|
}
|
|
|
|
function loadProvider(requestedProvider, requestedModel) {
|
|
const config = YAML.parse(fs.readFileSync(CONFIG_PATH, "utf8"));
|
|
const providerId = requestedProvider || config.defaultProvider;
|
|
const provider = config.providers[providerId];
|
|
if (!provider) throw new Error(`Unknown provider ${providerId}`);
|
|
const apiKey =
|
|
String(provider.apiKey || "").trim() ||
|
|
process.env[String(provider.apiKeyEnv || "")] ||
|
|
"";
|
|
if (!apiKey) throw new Error(`No API key configured for ${providerId}`);
|
|
const defaultModel = String(config.defaultModel || "").split(":").pop();
|
|
const model =
|
|
requestedModel || provider.models?.default || defaultModel;
|
|
return {
|
|
providerId,
|
|
model,
|
|
apiKey,
|
|
baseURL: String(provider.baseURL).replace(/\/+$/, ""),
|
|
};
|
|
}
|
|
|
|
function compactEvidence(payload, maxSurfaces = 80) {
|
|
const geometry = payload.geometry_evidence || {};
|
|
const design = payload.design_ir || {};
|
|
return {
|
|
evidence_kind: "final_brep_observation_not_source_history",
|
|
geometry: {
|
|
solid_count: geometry.solid_count,
|
|
face_count: geometry.face_count,
|
|
volume: geometry.volume,
|
|
bounding_box: geometry.bounding_box,
|
|
analytic_surface_counts: geometry.analytic_surface_counts,
|
|
analytic_surfaces: (geometry.analytic_surfaces || []).slice(0, maxSurfaces),
|
|
},
|
|
inferred_hypotheses: {
|
|
family: design.part_family,
|
|
features: design.features,
|
|
constraints: design.constraints,
|
|
semantic_parameter_candidates: design.semantic_parameter_candidates,
|
|
},
|
|
};
|
|
}
|
|
|
|
function reconstructionPrompt(evidence, previous, feedback, experience = []) {
|
|
const example = readJson(EXAMPLE_PATH);
|
|
return [
|
|
"You are Agent A, a STEP reverse-engineering program synthesizer.",
|
|
"Generate an independent executable DesignIR 2.0 JSON object from final-BRep evidence.",
|
|
"The source STEP is unavailable. Never reference, import, embed, or describe it as a base feature.",
|
|
"STEP does not contain source history. Use canonical reconstruction order and explicitly inferred semantic names.",
|
|
"Current executable operations are exactly: extrude_circle, extrude_rectangle, add_cylinder, through_hole, polar_hole_pattern.",
|
|
"Cylinder and extrusion primitives may use any observed axis. Hole cutters currently require the positive Z axis.",
|
|
"Preserve the observed absolute coordinate frame and orientation.",
|
|
"For cylinder and extrusion features, set anchor to geometric_center and center to the observed 3D bounding-box center unless there is strong evidence for another placement.",
|
|
"The legacy/default anchor is base_center; never rely on that default for reverse engineering.",
|
|
"A value reference object may contain exactly one key: parameter or expression. Do not add scale or arithmetic keys.",
|
|
"Do not expose an editable parameter unless at least one feature directly references it. Non-driving measurements must be editable false or omitted.",
|
|
"Use named editable parameters such as plate_thickness, hole_diameter, hole_count, bolt_circle_diameter, bore_diameter, overall_length, overall_width.",
|
|
"Every editable parameter must be referenced by a feature.",
|
|
"Every perturbation must contain parameter, scale, expected_change, and preserve.",
|
|
"Use an empty list for constraints when none are justified.",
|
|
"Return JSON only, without Markdown.",
|
|
"",
|
|
"REFERENCE FORMAT:",
|
|
JSON.stringify(example),
|
|
"",
|
|
"PRIVATE GEOMETRY EVIDENCE:",
|
|
JSON.stringify(evidence),
|
|
experience.length
|
|
? `\nVALIDATED CANDIDATE METHODS (apply only when relevant):\n${JSON.stringify(experience)}`
|
|
: "",
|
|
previous ? `\nPREVIOUS CANDIDATE:\n${JSON.stringify(previous)}` : "",
|
|
feedback ? `\nVALIDATOR FEEDBACK:\n${feedback}` : "",
|
|
]
|
|
.filter(Boolean)
|
|
.join("\n");
|
|
}
|
|
|
|
async function callModel(provider, prompt) {
|
|
const response = await fetch(`${provider.baseURL}/chat/completions`, {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Bearer ${provider.apiKey}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({
|
|
model: provider.model,
|
|
temperature: 0.2,
|
|
messages: [
|
|
{
|
|
role: "system",
|
|
content:
|
|
"Produce normalized, editable parametric CAD design states. Output strict JSON.",
|
|
},
|
|
{ role: "user", content: prompt },
|
|
],
|
|
response_format: { type: "json_object" },
|
|
}),
|
|
signal: AbortSignal.timeout(90_000),
|
|
});
|
|
if (!response.ok) {
|
|
const detail = (await response.text()).slice(0, 600);
|
|
throw new Error(`LLM HTTP ${response.status}: ${detail}`);
|
|
}
|
|
const payload = await response.json();
|
|
const content = payload.choices?.[0]?.message?.content;
|
|
if (!content) throw new Error("LLM returned no message content");
|
|
const cleaned = String(content)
|
|
.replace(/^```(?:json)?\s*/i, "")
|
|
.replace(/\s*```$/, "");
|
|
return {
|
|
designir: JSON.parse(cleaned),
|
|
usage: payload.usage || {},
|
|
};
|
|
}
|
|
|
|
function runPython(arguments_) {
|
|
const result = spawnSync(PYTHON, [DESIGNIR_CLI, ...arguments_], {
|
|
cwd: REPO,
|
|
encoding: "utf8",
|
|
timeout: 120_000,
|
|
});
|
|
const output = `${result.stdout || ""}\n${result.stderr || ""}`.trim();
|
|
return { ok: result.status === 0, status: result.status, output };
|
|
}
|
|
|
|
async function runOne(job, provider, maxAttempts, options = {}) {
|
|
const shortId = job.id.slice(0, 16);
|
|
const root = path.join(
|
|
options.attemptsRoot || path.join(TRAJECTORY_DIR, "attempts"),
|
|
shortId,
|
|
);
|
|
fs.mkdirSync(root, { recursive: true });
|
|
const evidencePayload = readJson(job.evidence);
|
|
const evidence = compactEvidence(evidencePayload);
|
|
const unsupportedSurfaces = Object.entries(
|
|
evidence.geometry.analytic_surface_counts || {},
|
|
)
|
|
.filter(([kind, count]) => count > 0 && !["plane", "cylinder"].includes(kind))
|
|
.map(([kind]) => kind);
|
|
if (unsupportedSurfaces.length) {
|
|
const trajectory = {
|
|
schema_version: "1.0",
|
|
trajectory_kind: "execution_validated_step_to_designir",
|
|
case_id: job.id,
|
|
family: job.family,
|
|
provider: provider.providerId,
|
|
model: provider.model,
|
|
attempts: [],
|
|
state: "unsupported_feature_vocabulary",
|
|
missing_capabilities: unsupportedSurfaces.map(
|
|
(kind) => `reconstruct_${kind}_surface`,
|
|
),
|
|
};
|
|
writeJson(path.join(root, "trajectory.json"), trajectory);
|
|
return trajectory;
|
|
}
|
|
let previous = null;
|
|
let feedback = "";
|
|
const attempts = [];
|
|
for (let index = 1; index <= maxAttempts; index += 1) {
|
|
const started = Date.now();
|
|
try {
|
|
const generated = await callModel(
|
|
provider,
|
|
reconstructionPrompt(
|
|
evidence,
|
|
previous,
|
|
feedback,
|
|
options.experience || [],
|
|
),
|
|
);
|
|
const designir = generated.designir;
|
|
designir.model_id ||= `trajectory_${shortId}_${index}`;
|
|
designir.applied_experience_ids = (options.experience || []).map(
|
|
(method) => method.id,
|
|
);
|
|
const designirPath = path.join(root, `attempt-${index}.designir.json`);
|
|
const rebuiltPath = path.join(root, `attempt-${index}.step`);
|
|
const reportPath = path.join(root, `attempt-${index}.acceptance.json`);
|
|
writeJson(designirPath, designir);
|
|
const rebuild = runPython([
|
|
"isolated-rebuild",
|
|
designirPath,
|
|
"--output",
|
|
rebuiltPath,
|
|
"--backend",
|
|
"build123d",
|
|
]);
|
|
if (!rebuild.ok) {
|
|
feedback = `Isolated rebuild failed: ${rebuild.output}`;
|
|
attempts.push({
|
|
index,
|
|
state: "compile_failed",
|
|
duration_ms: Date.now() - started,
|
|
usage: generated.usage,
|
|
feedback,
|
|
});
|
|
previous = designir;
|
|
continue;
|
|
}
|
|
const acceptance = runPython([
|
|
"accept",
|
|
"--teacher",
|
|
job.teacher,
|
|
"--designir",
|
|
designirPath,
|
|
"--rebuilt",
|
|
rebuiltPath,
|
|
"--report",
|
|
reportPath,
|
|
]);
|
|
const report = fs.existsSync(reportPath) ? readJson(reportPath) : null;
|
|
const state =
|
|
acceptance.ok && report?.verdict === "pass"
|
|
? "replay_validated"
|
|
: "acceptance_failed";
|
|
attempts.push({
|
|
index,
|
|
state,
|
|
duration_ms: Date.now() - started,
|
|
usage: generated.usage,
|
|
report: reportPath,
|
|
});
|
|
if (state === "replay_validated") {
|
|
const trajectory = {
|
|
schema_version: "1.0",
|
|
trajectory_kind: "execution_validated_step_to_designir",
|
|
case_id: job.id,
|
|
split: job.split,
|
|
family: job.family,
|
|
provider: provider.providerId,
|
|
model: provider.model,
|
|
selected_attempt: index,
|
|
designir: designirPath,
|
|
rebuilt_step: rebuiltPath,
|
|
acceptance_report: reportPath,
|
|
attempts,
|
|
state,
|
|
};
|
|
writeJson(path.join(root, "trajectory.json"), trajectory);
|
|
return trajectory;
|
|
}
|
|
feedback = `Acceptance failed: ${JSON.stringify({
|
|
threshold_checks: report?.threshold_checks,
|
|
geometry: report?.geometry
|
|
? {
|
|
relative_volume_error: report.geometry.relative_volume_error,
|
|
bounding_box_axis_error_mm:
|
|
report.geometry.bounding_box_axis_error_mm,
|
|
symmetric_difference_ratio:
|
|
report.geometry.symmetric_difference_ratio,
|
|
teacher_facts: report.geometry.teacher,
|
|
rebuilt_facts: report.geometry.rebuilt,
|
|
}
|
|
: null,
|
|
editability: report?.editability,
|
|
})}`;
|
|
previous = designir;
|
|
} catch (error) {
|
|
feedback = error instanceof Error ? error.message : String(error);
|
|
attempts.push({
|
|
index,
|
|
state: "agent_failed",
|
|
duration_ms: Date.now() - started,
|
|
feedback,
|
|
});
|
|
}
|
|
}
|
|
const trajectory = {
|
|
schema_version: "1.0",
|
|
trajectory_kind: "execution_validated_step_to_designir",
|
|
case_id: job.id,
|
|
family: job.family,
|
|
provider: provider.providerId,
|
|
model: provider.model,
|
|
attempts,
|
|
state: "quarantine",
|
|
};
|
|
writeJson(path.join(root, "trajectory.json"), trajectory);
|
|
return trajectory;
|
|
}
|
|
|
|
function summarize() {
|
|
const root = path.join(TRAJECTORY_DIR, "attempts");
|
|
const rows = fs.existsSync(root)
|
|
? fs
|
|
.readdirSync(root)
|
|
.map((name) => path.join(root, name, "trajectory.json"))
|
|
.filter((file) => fs.existsSync(file))
|
|
.map(readJson)
|
|
: [];
|
|
const states = {};
|
|
for (const row of rows) states[row.state] = (states[row.state] || 0) + 1;
|
|
return {
|
|
schema_version: "1.0",
|
|
report_kind: "trajectory_run_summary",
|
|
case_count: rows.length,
|
|
states,
|
|
validated_trajectory_count: states.replay_validated || 0,
|
|
quarantined_count: states.quarantine || 0,
|
|
rows: rows.map((row) => ({
|
|
case_id: row.case_id,
|
|
family: row.family,
|
|
state: row.state,
|
|
attempt_count: row.attempts?.length || 0,
|
|
})),
|
|
};
|
|
}
|
|
|
|
function distillValidatedTrajectories() {
|
|
const root = path.join(TRAJECTORY_DIR, "attempts");
|
|
const trajectories = fs.existsSync(root)
|
|
? fs
|
|
.readdirSync(root)
|
|
.map((name) => path.join(root, name, "trajectory.json"))
|
|
.filter((file) => fs.existsSync(file))
|
|
.map(readJson)
|
|
.filter((row) => row.state === "replay_validated")
|
|
: [];
|
|
const groups = new Map();
|
|
for (const trajectory of trajectories) {
|
|
const designir = readJson(trajectory.designir);
|
|
const signature = designir.features
|
|
.map((feature) =>
|
|
["add_cylinder", "extrude_circle"].includes(feature.operation)
|
|
? "cylinder"
|
|
: feature.operation,
|
|
)
|
|
.sort()
|
|
.join("+");
|
|
const key = `${designir.family}::${signature}`;
|
|
const group = groups.get(key) || {
|
|
family: designir.family,
|
|
signature,
|
|
cases: [],
|
|
parameterNames: new Map(),
|
|
anchors: new Map(),
|
|
operations: new Set(),
|
|
};
|
|
group.cases.push(trajectory.case_id);
|
|
for (const name of Object.keys(designir.parameters || {})) {
|
|
group.parameterNames.set(name, (group.parameterNames.get(name) || 0) + 1);
|
|
}
|
|
for (const feature of designir.features || []) {
|
|
group.operations.add(feature.operation);
|
|
const anchor = feature.anchor || "base_center";
|
|
group.anchors.set(anchor, (group.anchors.get(anchor) || 0) + 1);
|
|
}
|
|
groups.set(key, group);
|
|
}
|
|
const candidates = [...groups.values()].map((group) => {
|
|
const simpleCylinder =
|
|
group.family === "simple_cylinder" && group.signature === "cylinder";
|
|
return {
|
|
id: `trajectory.${group.family}.${group.signature.replaceAll("+", "_")}`,
|
|
kind: "reconstruction_grammar",
|
|
promotion_state: "candidate",
|
|
applicability: simpleCylinder
|
|
? {
|
|
family: group.family,
|
|
evidence_conditions: [
|
|
"one dominant axis",
|
|
"one distinct coaxial radius",
|
|
"low topological complexity",
|
|
"solid volume approximately pi * radius^2 * axial_length",
|
|
],
|
|
}
|
|
: {
|
|
family: group.family,
|
|
operations: [...group.operations].sort(),
|
|
},
|
|
method: {
|
|
guidance: simpleCylinder
|
|
? "Interpret the single cylindrical surface as the outer body, not a bore. Build one cylinder from the observed outer radius, axial length, axis, and pose. Name the driving parameters outer_radius and overall_length. Preserve the absolute frame with an explicit center and anchor; do not expose duplicate width/thickness/inner-diameter parameters."
|
|
: "Reconstruct from analytic dimensions and axis evidence; preserve the observed absolute frame with an explicit center and anchor; expose only parameters that directly drive executable features.",
|
|
canonical_parameter_names: simpleCylinder
|
|
? ["outer_radius", "overall_length"]
|
|
: [],
|
|
rejected_parameter_hypotheses: simpleCylinder
|
|
? ["coaxial_inner_diameter", "duplicate_transverse_envelope_parameters"]
|
|
: [],
|
|
observed_parameter_names: Object.fromEntries(
|
|
[...group.parameterNames.entries()].sort(),
|
|
),
|
|
observed_anchor_modes: Object.fromEntries(
|
|
[...group.anchors.entries()].sort(),
|
|
),
|
|
},
|
|
validation: {
|
|
independent_replay_validated_case_count: new Set(group.cases).size,
|
|
source_hashes: [...new Set(group.cases)].sort(),
|
|
edit_contract_pass_required: true,
|
|
positive_heldout_ab_required: true,
|
|
heldout_ab_status: "not_run",
|
|
},
|
|
};
|
|
});
|
|
const payload = {
|
|
schema_version: "1.0",
|
|
library_kind: "execution_validated_trajectory_candidates",
|
|
status: candidates.length ? "candidates_only" : "empty",
|
|
policy: {
|
|
router_consumable: false,
|
|
automatic_publish_allowed: false,
|
|
minimum_independent_cases: 3,
|
|
positive_heldout_ab_required: true,
|
|
},
|
|
validated_trajectory_count: trajectories.length,
|
|
candidate_count: candidates.length,
|
|
candidates,
|
|
};
|
|
const output = path.join(TRAJECTORY_DIR, "candidate-experience.json");
|
|
writeJson(output, payload);
|
|
return { output, payload };
|
|
}
|
|
|
|
function sanitizedMethods(candidatePath) {
|
|
if (!candidatePath) return [];
|
|
return (readJson(candidatePath).candidates || []).map((candidate) => ({
|
|
id: candidate.id,
|
|
kind: candidate.kind,
|
|
applicability: candidate.applicability,
|
|
method: candidate.method,
|
|
}));
|
|
}
|
|
|
|
function benchmarkScore(rows) {
|
|
const passed = rows.filter((row) => row.state === "replay_validated").length;
|
|
return {
|
|
case_count: rows.length,
|
|
replay_validated_count: passed,
|
|
pass_rate: rows.length ? passed / rows.length : 0,
|
|
};
|
|
}
|
|
|
|
async function runBenchmark(args) {
|
|
const split = String(args.split || "validation");
|
|
if (!["validation", "test"].includes(split)) {
|
|
throw new Error("Benchmark split must be validation or test");
|
|
}
|
|
const limit = Math.max(1, Number(args.limit || 10));
|
|
const maxAttempts = Math.max(1, Number(args.attempts || 3));
|
|
const maxFaces = Math.max(1, Number(args["max-faces"] || 24));
|
|
const requestedFamily = String(args.family || "").trim();
|
|
const manifest = fs.existsSync(SPLIT_PATH) ? readJson(SPLIT_PATH) : initSplits();
|
|
const jobs = (manifest.splits?.[split] || [])
|
|
.filter((job) => !requestedFamily || job.family === requestedFamily)
|
|
.filter((job) => job.face_count <= maxFaces)
|
|
.slice(0, limit);
|
|
const provider = loadProvider(args.provider, args.model);
|
|
const candidatePath = String(
|
|
args.experience ||
|
|
path.join(TRAJECTORY_DIR, "candidate-experience.json"),
|
|
);
|
|
const experience = sanitizedMethods(candidatePath);
|
|
const benchmarkRoot = path.join(
|
|
TRAJECTORY_DIR,
|
|
"benchmarks",
|
|
`${split}-${requestedFamily || "all"}`,
|
|
);
|
|
const baseline = [];
|
|
const treatment = [];
|
|
for (const job of jobs) {
|
|
baseline.push(
|
|
await runOne({ ...job, split }, provider, maxAttempts, {
|
|
attemptsRoot: path.join(benchmarkRoot, "baseline"),
|
|
}),
|
|
);
|
|
treatment.push(
|
|
await runOne({ ...job, split }, provider, maxAttempts, {
|
|
attemptsRoot: path.join(benchmarkRoot, "experience"),
|
|
experience,
|
|
}),
|
|
);
|
|
}
|
|
const baselineScore = benchmarkScore(baseline);
|
|
const treatmentScore = benchmarkScore(treatment);
|
|
const report = {
|
|
schema_version: "1.0",
|
|
report_kind: "heldout_trajectory_ab",
|
|
split,
|
|
family: requestedFamily || "all",
|
|
provider: provider.providerId,
|
|
model: provider.model,
|
|
candidate_method_ids: experience.map((method) => method.id),
|
|
baseline: baselineScore,
|
|
experience: treatmentScore,
|
|
pass_rate_delta: treatmentScore.pass_rate - baselineScore.pass_rate,
|
|
positive: treatmentScore.pass_rate > baselineScore.pass_rate,
|
|
promotion_eligible:
|
|
jobs.length >= 10 && treatmentScore.pass_rate > baselineScore.pass_rate,
|
|
case_results: jobs.map((job, index) => ({
|
|
case_id: job.id,
|
|
baseline_state: baseline[index].state,
|
|
experience_state: treatment[index].state,
|
|
})),
|
|
};
|
|
writeJson(path.join(benchmarkRoot, "report.json"), report);
|
|
return report;
|
|
}
|
|
|
|
async function main() {
|
|
const args = argsObject(process.argv.slice(2));
|
|
const command = args._[0];
|
|
if (command === "init") {
|
|
const payload = initSplits();
|
|
process.stdout.write(
|
|
`${JSON.stringify({ split_path: SPLIT_PATH, policy: payload.policy }, null, 2)}\n`,
|
|
);
|
|
return;
|
|
}
|
|
if (command === "summary") {
|
|
const report = summarize();
|
|
writeJson(path.join(TRAJECTORY_DIR, "summary.json"), report);
|
|
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
return;
|
|
}
|
|
if (command === "distill") {
|
|
const result = distillValidatedTrajectories();
|
|
process.stdout.write(
|
|
`${JSON.stringify(
|
|
{
|
|
output: result.output,
|
|
validated_trajectory_count: result.payload.validated_trajectory_count,
|
|
candidate_count: result.payload.candidate_count,
|
|
router_consumable: result.payload.policy.router_consumable,
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
);
|
|
return;
|
|
}
|
|
if (command === "benchmark") {
|
|
const report = await runBenchmark(args);
|
|
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
return;
|
|
}
|
|
if (command !== "run") {
|
|
throw new Error(
|
|
"Usage: trajectory_pipeline.mjs init|run|summary|distill|benchmark",
|
|
);
|
|
}
|
|
const split = String(args.split || "train");
|
|
const limit = Math.max(1, Number(args.limit || 5));
|
|
const maxAttempts = Math.max(1, Number(args.attempts || 3));
|
|
const maxFaces = Math.max(1, Number(args["max-faces"] || 24));
|
|
const requestedFamily = String(args.family || "").trim();
|
|
const rerun = Boolean(args.rerun);
|
|
const manifest = fs.existsSync(SPLIT_PATH) ? readJson(SPLIT_PATH) : initSplits();
|
|
const jobs = (manifest.splits?.[split] || [])
|
|
.filter((job) => !requestedFamily || job.family === requestedFamily)
|
|
.filter((job) => job.face_count <= maxFaces)
|
|
.filter(
|
|
(job) =>
|
|
rerun ||
|
|
!fs.existsSync(
|
|
path.join(
|
|
TRAJECTORY_DIR,
|
|
"attempts",
|
|
job.id.slice(0, 16),
|
|
"trajectory.json",
|
|
),
|
|
),
|
|
)
|
|
.slice(0, limit);
|
|
const provider = loadProvider(args.provider, args.model);
|
|
const results = [];
|
|
for (const job of jobs) {
|
|
const result = await runOne({ ...job, split }, provider, maxAttempts);
|
|
results.push({
|
|
case_id: job.id,
|
|
family: job.family,
|
|
face_count: job.face_count,
|
|
state: result.state,
|
|
});
|
|
process.stdout.write(`${JSON.stringify(results.at(-1))}\n`);
|
|
}
|
|
const report = summarize();
|
|
writeJson(path.join(TRAJECTORY_DIR, "summary.json"), report);
|
|
process.stdout.write(`${JSON.stringify({ completed: results.length, report }, null, 2)}\n`);
|
|
}
|
|
|
|
main().catch((error) => {
|
|
process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`);
|
|
process.exitCode = 1;
|
|
});
|