165 lines
10 KiB
Plaintext
165 lines
10 KiB
Plaintext
package com.linkerbot.vibemake.service.cad.ai;
|
|
|
|
import static org.assertj.core.api.Assertions.*;
|
|
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
|
import com.linkerbot.vibemake.service.config.CadAiProperties;
|
|
import java.io.InputStream;
|
|
import java.security.MessageDigest;
|
|
import java.util.ArrayList;
|
|
import java.util.HexFormat;
|
|
import java.util.List;
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
class CadAiSampleKnowledgeTest {
|
|
private final ObjectMapper json = new ObjectMapper();
|
|
|
|
@Test void sevenNewTypesRespectScopeFactsReferencesAndPublicationWithoutChangingGearHistory() throws Exception {
|
|
JsonNode suite;
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/evals/sample-parts-routing-v1.json")) {
|
|
suite = json.readTree(input);
|
|
}
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/evals/gear-routing-v8.json")) {
|
|
assertThat(HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(input.readAllBytes())))
|
|
.isEqualTo(suite.path("preserved_gear_v8_sha256").asText());
|
|
}
|
|
List<String> failures = new ArrayList<>();
|
|
for (JsonNode item : suite.path("cases")) {
|
|
CadAiProperties properties = new CadAiProperties();
|
|
properties.setKnowledgeMode(item.path("mode").asText("pilot"));
|
|
CadAiKnowledge knowledge = HistoricalCadAiKnowledge.knowledge(json, properties);
|
|
ObjectNode input = json.createObjectNode().put("query", item.path("query").asText());
|
|
input.set("facts", item.path("facts").deepCopy());
|
|
if (item.has("selected_type")) input.put("type_id", item.path("selected_type").asText());
|
|
ObjectNode before = input.deepCopy();
|
|
JsonNode result = knowledge.lookup(input, item.path("build").asText(suite.path("engine_build").asText()));
|
|
String id = item.path("id").asText();
|
|
String expectedStatus = HistoricalCadAiKnowledge.runtimeStatus(item);
|
|
if (!result.path("status").asText().equals(expectedStatus))
|
|
failures.add(id + ": expected " + expectedStatus + " but " + result);
|
|
if ("matched".equals(expectedStatus) && "matched".equals(result.path("status").asText())) {
|
|
assertThat(result.path("type_id").asText()).as(id).isEqualTo(item.path("type_id").asText());
|
|
assertThat(result.path("documents")).as(id).hasSize(1);
|
|
if (item.has("doc_id")) assertThat(result.path("documents").get(0).path("id").asText()).as(id).isEqualTo(item.path("doc_id").asText());
|
|
assertThat(result.path("documents").get(0).path("text").asText()).as(id).isNotBlank();
|
|
} else if (!"matched".equals(result.path("status").asText())) {
|
|
assertThat(result.path("documents")).as(id).isEmpty();
|
|
}
|
|
assertThat(input).as(id + " preserves supplied facts").isEqualTo(before);
|
|
}
|
|
assertThat(failures).isEmpty();
|
|
}
|
|
|
|
@Test void propertyDictionaryAndRepresentationChoicesStayCanonical() throws Exception {
|
|
JsonNode schema = HistoricalCadAiKnowledge.tools(json).definitions(true).stream()
|
|
.filter(tool -> "knowledge_lookup".equals(tool.name())).findFirst().orElseThrow()
|
|
.parameters().path("properties").path("facts");
|
|
assertThat(schema.path("properties").has("central_gap_mm")).isTrue();
|
|
assertThat(schema.path("properties").has("hole_count")).isTrue();
|
|
assertThat(schema.path("properties").size()).isLessThanOrEqualTo(64);
|
|
assertThat(schema.path("properties").path("geometry_representation").toString())
|
|
.contains("herringbone-continuous-midplane-v1", "double-helical-root-core-gap-v1", "hex-bolt-simplified-thread-v1");
|
|
}
|
|
|
|
@Test void absentFeaturesAndAdditionalCurrentPartScopesStayDistinct() throws Exception {
|
|
JsonNode suite;
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/evals/sample-parts-routing-v2.json")) {
|
|
suite = json.readTree(input);
|
|
}
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/evals/sample-parts-routing-v1.json")) {
|
|
assertThat(HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(input.readAllBytes())))
|
|
.isEqualTo(suite.path("preserved_v1_sha256").asText());
|
|
}
|
|
JsonNode additional;
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/evals/sample-parts-routing-v4.json")) {
|
|
additional = json.readTree(input);
|
|
}
|
|
CadAiProperties properties = new CadAiProperties();
|
|
properties.setKnowledgeMode("pilot");
|
|
CadAiKnowledge knowledge = HistoricalCadAiKnowledge.knowledge(json, properties);
|
|
for (JsonNode fixture : List.of(suite, additional)) for (JsonNode item : fixture.path("cases")) {
|
|
ObjectNode input = json.createObjectNode().put("query", item.path("query").asText());
|
|
if (item.has("selected_type")) input.put("type_id", item.path("selected_type").asText());
|
|
input.set("facts", item.path("facts").deepCopy());
|
|
ObjectNode before = input.deepCopy();
|
|
JsonNode result = knowledge.lookup(input, suite.path("engine_build").asText());
|
|
assertThat(result.path("status").asText()).as(item.path("id").asText())
|
|
.isEqualTo(item.path("status").asText());
|
|
if ("matched".equals(item.path("status").asText())) {
|
|
assertThat(result.path("documents")).hasSize(1);
|
|
assertThat(result.path("documents").get(0).path("id").asText()).isEqualTo(item.path("doc_id").asText());
|
|
} else assertThat(result.path("documents")).isEmpty();
|
|
assertThat(input).isEqualTo(before);
|
|
}
|
|
}
|
|
|
|
@Test void selectedPartRecognizesQualifiedPairNamesWithoutAbsorbingOtherTargets() throws Exception {
|
|
JsonNode suite;
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/evals/sample-parts-routing-v3.json")) {
|
|
suite = json.readTree(input);
|
|
}
|
|
CadAiProperties properties = new CadAiProperties();
|
|
properties.setKnowledgeMode("pilot");
|
|
CadAiKnowledge knowledge = HistoricalCadAiKnowledge.knowledge(json, properties);
|
|
List<String> failures = new ArrayList<>();
|
|
for (JsonNode item : suite.path("cases")) {
|
|
ObjectNode input = json.createObjectNode().put("query", item.path("query").asText());
|
|
if (item.has("selected_type")) input.put("type_id", item.path("selected_type").asText());
|
|
input.set("facts", item.path("facts").deepCopy());
|
|
ObjectNode before = input.deepCopy();
|
|
JsonNode result = knowledge.lookup(input, suite.path("engine_build").asText());
|
|
String id = item.path("id").asText();
|
|
if (!result.path("status").asText().equals(item.path("status").asText())) {
|
|
failures.add(id + ": expected " + item.path("status").asText() + " but " + result);
|
|
continue;
|
|
}
|
|
if ("matched".equals(item.path("status").asText())) {
|
|
assertThat(result.path("type_id").asText()).as(id).isEqualTo(item.path("type_id").asText());
|
|
assertThat(result.path("documents")).as(id).hasSize(1);
|
|
assertThat(result.path("documents").get(0).path("id").asText()).as(id).isEqualTo(item.path("doc_id").asText());
|
|
} else assertThat(result.path("documents")).as(id).isEmpty();
|
|
for (String field : List.of("missing", "conflicts")) {
|
|
if (!item.has(field)) continue;
|
|
List<String> expected = new ArrayList<>();
|
|
for (JsonNode value : item.path(field)) expected.add(value.asText());
|
|
List<String> actual = new ArrayList<>();
|
|
for (JsonNode value : result.path(field)) actual.add(value.asText());
|
|
assertThat(actual).as(id + " " + field).containsExactlyInAnyOrderElementsOf(expected);
|
|
}
|
|
assertThat(input).as(id + " preserves supplied facts").isEqualTo(before);
|
|
}
|
|
assertThat(failures).isEmpty();
|
|
}
|
|
|
|
@Test void currentPartReferencesAndInterfaceDimensionsDoNotBecomeAdditionalTargets() throws Exception {
|
|
JsonNode suite;
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/evals/sample-parts-routing-v5.json")) {
|
|
suite = json.readTree(input);
|
|
}
|
|
CadAiProperties properties = new CadAiProperties();
|
|
properties.setKnowledgeMode("pilot");
|
|
CadAiKnowledge knowledge = HistoricalCadAiKnowledge.knowledge(json, properties);
|
|
List<String> failures = new ArrayList<>();
|
|
for (JsonNode item : suite.path("cases")) {
|
|
ObjectNode input = json.createObjectNode().put("query", item.path("query").asText());
|
|
if (item.has("selected_type")) input.put("type_id", item.path("selected_type").asText());
|
|
input.set("facts", item.path("facts").deepCopy());
|
|
ObjectNode before = input.deepCopy();
|
|
JsonNode result = knowledge.lookup(input, null);
|
|
String id = item.path("id").asText();
|
|
if (!result.path("status").asText().equals(item.path("status").asText()))
|
|
failures.add(id + ": expected " + item.path("status").asText() + " but " + result);
|
|
if ("matched".equals(result.path("status").asText())) {
|
|
assertThat(result.path("type_id").asText()).as(id).isEqualTo(item.path("selected_type").asText());
|
|
assertThat(result.path("documents")).as(id).hasSize(1);
|
|
} else assertThat(result.path("documents")).as(id).isEmpty();
|
|
for (JsonNode field : item.path("missing"))
|
|
assertThat(result.path("missing").toString()).as(id).contains(field.asText());
|
|
assertThat(input).as(id + " never extracts or corrects supplied facts").isEqualTo(before);
|
|
}
|
|
assertThat(failures).isEmpty();
|
|
}
|
|
}
|