208 lines
12 KiB
Plaintext
208 lines
12 KiB
Plaintext
package com.linkerbot.vibemake.service.cad.ai;
|
|
|
|
import static org.assertj.core.api.Assertions.*;
|
|
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
|
import com.linkerbot.vibemake.service.config.CadAiProperties;
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.util.*;
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
/** Live policy tests; historical fixtures run separately against their frozen policy. */
|
|
class CadAiKnowledgeUtilityTest {
|
|
private static final String BUILD = "f3d62ce93f72db42f311a46e";
|
|
private final ObjectMapper json = new ObjectMapper();
|
|
|
|
private JsonNode resource(String path) throws IOException {
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/" + path)) {
|
|
if (input == null) throw new IOException("Missing resource: " + path);
|
|
return json.readTree(input);
|
|
}
|
|
}
|
|
|
|
private CadAiKnowledge live(String mode) throws IOException {
|
|
CadAiProperties properties = new CadAiProperties();
|
|
properties.setKnowledgeMode(mode);
|
|
return new CadAiKnowledge(json, properties);
|
|
}
|
|
|
|
private ObjectNode request(String query) {
|
|
return json.createObjectNode().put("query", query);
|
|
}
|
|
|
|
private ObjectNode completeRequest(JsonNode document) throws IOException {
|
|
String id = document.path("doc_id").asText();
|
|
if ("housing-y-bore".equals(id)) return request("创建带筋轴承座,用XZ草图造筋");
|
|
for (String suite : List.of("gear-routing-v8.json", "sample-parts-routing-v1.json")) {
|
|
for (JsonNode item : resource("evals/" + suite).path("cases")) {
|
|
if (!id.equals(item.path("doc_id").asText()) || !"matched".equals(item.path("status").asText()) || !item.has("facts")) continue;
|
|
ObjectNode input = request(item.path("query").asText()).put("type_id", document.path("type_id").asText());
|
|
input.set("facts", item.path("facts").deepCopy());
|
|
return input;
|
|
}
|
|
}
|
|
throw new IOException("No existing matched fixture for retained recipe: " + id);
|
|
}
|
|
|
|
@Test void preservesExistingGearRecipesAndOnlyPrunesBasicSamples() throws Exception {
|
|
JsonNode policy = resource("generation/policy.json");
|
|
Map<String, JsonNode> previous = new HashMap<>();
|
|
for (JsonNode document : HistoricalCadAiKnowledge.policy(json).path("documents"))
|
|
previous.put(document.path("doc_id").asText(), document);
|
|
int active = 0;
|
|
int retired = 0;
|
|
int recipes = 0;
|
|
int hints = 0;
|
|
int currentCharacters = 0;
|
|
int priorCharacters = 0;
|
|
for (JsonNode document : policy.path("documents")) {
|
|
if ("retired".equals(document.path("status").asText())) { retired++; continue; }
|
|
active++;
|
|
assertThat(document.path("status").asText()).isEqualTo("experimental");
|
|
if ("recipe".equals(document.path("content_role").asText())) {
|
|
recipes++;
|
|
JsonNode original = previous.get(document.path("doc_id").asText());
|
|
for (String field : List.of("path", "version", "sha256", "type_id", "conditions", "compatible_builds", "evidence"))
|
|
assertThat(document.path(field)).as(document.path("doc_id").asText() + " preserves " + field).isEqualTo(original.path(field));
|
|
} else {
|
|
hints++;
|
|
assertThat(document.path("doc_id").asText()).isEqualTo("housing-y-bore");
|
|
}
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/" + document.path("path").asText())) {
|
|
byte[] bytes = input.readAllBytes();
|
|
CadAiKnowledge.verifyHash(bytes, document.path("sha256").asText());
|
|
currentCharacters += new String(bytes, StandardCharsets.UTF_8).length();
|
|
}
|
|
if ("risk_hint".equals(document.path("content_role").asText()))
|
|
assertThat(document.path("evidence").path("current_text_model_effect").asText()).startsWith("not_tested");
|
|
}
|
|
for (JsonNode document : HistoricalCadAiKnowledge.policy(json).path("documents")) {
|
|
if ("retired".equals(document.path("status").asText())) continue;
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/knowledge/" + document.path("path").asText())) {
|
|
priorCharacters += new String(input.readAllBytes(), StandardCharsets.UTF_8).length();
|
|
}
|
|
}
|
|
assertThat(active).isEqualTo(16);
|
|
assertThat(retired).isEqualTo(5);
|
|
assertThat(recipes).isEqualTo(15);
|
|
assertThat(hints).isEqualTo(1);
|
|
assertThat(currentCharacters).isLessThan(priorCharacters).isGreaterThan(priorCharacters / 2);
|
|
}
|
|
|
|
@Test void withdrawnTypesReturnNoBodyAndDoNotRequestConditions() throws Exception {
|
|
CadAiKnowledge knowledge = live("pilot");
|
|
Set<String> withdrawn = new HashSet<>();
|
|
for (JsonNode document : resource("generation/policy.json").path("documents"))
|
|
if ("retired".equals(document.path("status").asText()) && document.has("type_id")) withdrawn.add(document.path("type_id").asText());
|
|
Set<String> covered = new HashSet<>();
|
|
for (String suite : List.of("gear-routing-v8.json", "sample-parts-routing-v1.json")) {
|
|
for (JsonNode item : resource("evals/" + suite).path("cases")) {
|
|
String type = item.path("type_id").asText();
|
|
if (!"matched".equals(item.path("status").asText()) || !withdrawn.contains(type)) continue;
|
|
ObjectNode input = request(item.path("query").asText()).put("type_id", type);
|
|
input.set("facts", item.path("facts"));
|
|
JsonNode result = knowledge.lookup(input, BUILD);
|
|
assertThat(result.path("status").asText()).as(item.path("id").asText()).isEqualTo("uncovered");
|
|
assertThat(result.path("documents")).isEmpty();
|
|
assertThat(result.path("missing")).isEmpty();
|
|
covered.add(type);
|
|
}
|
|
}
|
|
assertThat(covered).containsExactlyInAnyOrderElementsOf(withdrawn);
|
|
}
|
|
|
|
@Test void housingHintOnlyActivatesForCurrentXzRibOperationWithoutDimensionQuestions() throws Exception {
|
|
CadAiKnowledge knowledge = live("pilot");
|
|
for (String query : List.of("创建带筋轴承座", "创建带筋轴承座,用XY草图造筋", "创建带筋轴承座,XZ草图开孔", "创建带筋轴承座,用XYZ草图造筋")) {
|
|
JsonNode result = knowledge.lookup(request(query), BUILD);
|
|
assertThat(result.path("status").asText()).as(query).isEqualTo("not_applicable");
|
|
assertThat(result.path("documents")).isEmpty();
|
|
assertThat(result.path("missing")).isEmpty();
|
|
}
|
|
for (String query : List.of("创建带筋轴承座,用XZ草图造筋", "create ribbed bearing housing with ribs sketched on XZ")) {
|
|
JsonNode result = knowledge.lookup(request(query), BUILD);
|
|
assertThat(result.path("status").asText()).as(query).isEqualTo("matched");
|
|
assertThat(result.path("documents").get(0).path("id").asText()).isEqualTo("housing-y-bore");
|
|
assertThat(result.path("documents").get(0).path("content_role").asText()).isEqualTo("risk_hint");
|
|
assertThat(result.path("missing")).isEmpty();
|
|
}
|
|
}
|
|
|
|
@Test void activationDoesNotOverrideWrongTypesNegationOrReferenceScope() throws Exception {
|
|
CadAiKnowledge knowledge = live("pilot");
|
|
for (ObjectNode input : List.of(
|
|
request("创建安装板,用XZ草图造筋").put("type_id", "ribbed-bearing-housing"),
|
|
request("创建带筋轴承座,不用XZ草图造筋"),
|
|
request("不创建XZ带筋轴承座,只创建安装板"),
|
|
request("参考XZ带筋轴承座,当前待建部件是台阶轴"))) {
|
|
JsonNode result = knowledge.lookup(input, BUILD);
|
|
assertThat(result.path("status").asText()).as(input.toString()).isNotEqualTo("matched");
|
|
assertThat(result.path("documents")).isEmpty();
|
|
}
|
|
}
|
|
|
|
@Test void activeDocumentsExposeTheirExactNewVersionHashAndRole() throws Exception {
|
|
CadAiKnowledge knowledge = live("pilot");
|
|
CadAiTools tools = new CadAiTools(json);
|
|
for (JsonNode document : resource("generation/policy.json").path("documents")) {
|
|
if ("retired".equals(document.path("status").asText())) continue;
|
|
ObjectNode input = completeRequest(document);
|
|
tools.validateCall("knowledge_lookup", input);
|
|
ObjectNode before = input.deepCopy();
|
|
JsonNode result = knowledge.lookup(input, BUILD);
|
|
assertThat(result.path("status").asText()).as(document.path("doc_id").asText()).isEqualTo("matched");
|
|
JsonNode returned = result.path("documents").get(0);
|
|
for (String key : List.of("version", "sha256", "status", "content_role"))
|
|
assertThat(returned.path(key)).isEqualTo(document.path(key));
|
|
CadAiKnowledge.verifyHash(returned.path("text").asText().getBytes(StandardCharsets.UTF_8), document.path("sha256").asText());
|
|
assertThat(input).isEqualTo(before);
|
|
for (String build : Arrays.asList(null, "", "0".repeat(24), "independent-frontend-release", "745a1240964583102b5bdc4f")) {
|
|
JsonNode independent = knowledge.lookup(input, build);
|
|
assertThat(independent.path("status").asText()).isEqualTo("matched");
|
|
assertThat(independent.path("documents")).isEqualTo(result.path("documents"));
|
|
}
|
|
assertThat(live("enabled").lookup(input, BUILD).path("documents")).isEmpty();
|
|
assertThat(live("off").lookup(input, BUILD).path("status").asText()).isEqualTo("disabled");
|
|
}
|
|
}
|
|
|
|
@Test void schemaDoesNotAdvertiseWithdrawnRepresentationsAndGenericSkillsStayIdentical() throws Exception {
|
|
JsonNode schema = new CadAiTools(json).definitions(true).stream().filter(tool -> "knowledge_lookup".equals(tool.name()))
|
|
.findFirst().orElseThrow().parameters();
|
|
List<String> representations = new ArrayList<>();
|
|
for (JsonNode value : schema.path("properties").path("facts").path("properties").path("geometry_representation").path("enum"))
|
|
if (value.isTextual()) representations.add(value.asText());
|
|
Set<String> expected = new HashSet<>();
|
|
Set<String> basic = Set.of("plate-through-holes", "shaft-cumulative-segments", "hex-bolt-simplified", "housing-y-bore", "enclosure-shared-interface");
|
|
for (JsonNode document : HistoricalCadAiKnowledge.policy(json).path("documents")) {
|
|
if ("retired".equals(document.path("status").asText()) || basic.contains(document.path("doc_id").asText())) continue;
|
|
JsonNode value = document.path("conditions").path("geometry_representation").path("equals");
|
|
if (value.isTextual()) expected.add(value.asText());
|
|
}
|
|
assertThat(representations).containsExactlyInAnyOrderElementsOf(expected);
|
|
assertThat(representations).doesNotContain("ribbed-housing-y-bore-z-mount-v1", "hex-bolt-simplified-thread-v1");
|
|
try (InputStream input = getClass().getResourceAsStream("/cad/ai/skills.json")) {
|
|
CadAiKnowledge.verifyHash(input.readAllBytes(), "026284eff26c8b4be1f7864219921a54828338973317c291f7e862b1cc452ef7");
|
|
}
|
|
}
|
|
|
|
@Test void rejectsMalformedActivationsAndUnguardedEmptyRecipes() throws Exception {
|
|
JsonNode original = resource("generation/policy.json");
|
|
for (String badActivation : List.of("{}", "{\"all_terms\":[],\"any_terms\":[\"筋\"]}",
|
|
"{\"all_terms\":[\"XZ\"],\"any_terms\":[\"筋\"]}", "{\"all_terms\":[\"xz\"],\"any_terms\":[\"\"]}")) {
|
|
JsonNode policy = original.deepCopy();
|
|
ObjectNode housing = (ObjectNode) policy.path("documents").get(17);
|
|
assertThat(housing.path("doc_id").asText()).isEqualTo("housing-y-bore");
|
|
housing.set("activation", json.readTree(badActivation));
|
|
assertThatThrownBy(() -> new CadAiKnowledge(json, new CadAiProperties(), policy)).isInstanceOf(IOException.class);
|
|
}
|
|
JsonNode policy = original.deepCopy();
|
|
((ObjectNode) policy.path("documents").get(17)).remove("activation");
|
|
assertThatThrownBy(() -> new CadAiKnowledge(json, new CadAiProperties(), policy)).isInstanceOf(IOException.class);
|
|
}
|
|
}
|