Files
vibemake-pcb-studio/tools/libraries/upload.mjs
T
likang 7e01147566
CI / typecheck + test + build (push) Has been cancelled
Initial commit: import VibeMake PCB Studio
2026-11-03 11:20:08 +08:00

192 lines
8.7 KiB
JavaScript

/**
* Upload the complete official symbol + footprint libraries to R2.
*
* Sources (gitignored): kicad-symbols-src/ and kicad-footprints-src/ (upstream
* library repos, master). Upstream symbols moved to the one-symbol-per-file
* layout (<Lib>.kicad_symdir/*.kicad_sym); the app, like released KiCad,
* loads one .kicad_sym per library, so each dir is merged into a single
* library file. The merge is lossless: every top-level `(symbol …)` block is
* copied byte-for-byte via a balanced-paren scan, never reformatted.
*
* Uploads:
* symbols/<Lib>.kicad_sym + symbols/index.json [{name,count,symbols}]
* footprints/<Lib>.pretty/<FP>.kicad_mod + footprints/index.json
* [{name,footprints,pads,descr,tags}]
*
* Usage: R2_* env vars (see tools/r2.mjs), then `node tools/libraries/upload.mjs`.
*/
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { gzipSync } from 'node:zlib';
import { putObject, uploadAll } from '../r2.mjs';
import { perSymbolFiles, stagedFileName, topLevelSymbols, unitCountOf, wrapLib } from './split.mjs';
import { footprintIndexInfo } from './fp_index.mjs';
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
/**
* Store one index gzipped, and say so.
*
* These two files are what every chooser needs before it can draw a row, and
* they are the only objects in the bucket that every session fetches. Raw they
* are 357 kB (symbols) and 649 kB (footprints); gzipped, 80 kB and 83 kB — 4.4x
* and 7.8x, because a list of library and part names is almost all repeated
* substrings. `content-encoding` makes the browser decompress transparently, so
* nothing on the app side changes.
*
* A day of `cache-control` with `stale-while-revalidate`: the set changes when
* upstream KiCad's libraries do, which is not daily, and the app revalidates
* against the ETag on its own anyway (`libraryHosts.ts`). `must-revalidate` is
* deliberately absent — a stale index for one load is a chooser missing a part
* that landed yesterday, and blocking on the network instead is the lag this
* whole change is about.
*/
async function putIndex(key, value) {
const raw = Buffer.from(`${JSON.stringify(value)}\n`);
const gz = gzipSync(raw, { level: 9 });
await putObject(key, gz, 'application/json', {
'content-encoding': 'gzip',
'cache-control': 'public, max-age=86400, stale-while-revalidate=604800',
});
console.log(`${key}: ${raw.length} -> ${gz.length} bytes gzipped`);
}
const SYM_SRC = join(ROOT, 'kicad-symbols-src');
const FP_SRC = join(ROOT, 'kicad-footprints-src');
// --- symbols: merge each .kicad_symdir into one library file ------------------
const symEntries = [];
const symIndex = [];
const symDirs = readdirSync(SYM_SRC).filter((d) => d.endsWith('.kicad_symdir'));
for (const dir of symDirs.sort()) {
const lib = dir.replace(/\.kicad_symdir$/, '');
const parts = [];
for (const f of readdirSync(join(SYM_SRC, dir)).sort()) {
if (!f.endsWith('.kicad_sym')) continue;
for (const s of topLevelSymbols(readFileSync(join(SYM_SRC, dir, f), 'utf8'))) parts.push(s);
}
// Derived symbols (`extends`) must appear after their parent; upstream keeps
// parents and derivatives in one file, and file order preserves that.
const names = parts.map((p) => p.name);
// LIB_SYMBOL::IsPower, carried in the index so the chooser's power filter is
// exact without loading every library. A derived symbol inherits it from the
// symbol it extends, so the parents are resolved first.
const blockByName = new Map(parts.map((p) => [p.name, p.block]));
const isPower = (name, seen = new Set()) => {
const block = blockByName.get(name);
if (!block || seen.has(name)) return false;
if (/\(\s*power\s*\)/.test(block)) return true;
const ext = /\(\s*extends\s+"([^"]+)"/.exec(block);
return ext ? isPower(ext[1], seen.add(name)) : false;
};
const power = names.filter((n) => isPower(n));
const body = parts.map((p) => `\t${p.block}`).join('\n');
const merged = wrapLib('ziro_library_merge', body);
symEntries.push([`symbols/${lib}.kicad_sym`, Buffer.from(merged), 'text/plain']);
// --- one file per symbol ----------------------------------------------------
// Placing a part used to fetch its whole library: 7.0 MB for one connector
// out of Connector_Generic, 5.2 MB out of Connector. These are the same
// blocks, byte for byte, split so the app can ask for exactly the symbol it
// is placing. The merged library above stays, for the library browser (which
// legitimately wants all of a library) and as the fallback.
//
// A derived symbol carries no geometry of its own, so its file must also hold
// the chain it extends. `LIB_SYMBOL::Flatten` (our `resolveExtends`) looks the
// parent up **by name within the same file** and, finding nothing, silently
// keeps the child's own empty body: the symbol parses, places, and has no
// pins. Parents are emitted before the child, matching the order the merged
// file relies on.
const oneDir = join(ROOT, 'tools/libraries/out/symbols', lib);
mkdirSync(oneDir, { recursive: true });
for (const { name, text } of perSymbolFiles(parts)) {
// The key holds the symbol name raw; the app percent-encodes it into the
// request path, which the object store decodes back to this key.
symEntries.push([`symbols/${lib}/${name}.kicad_sym`, Buffer.from(text), 'text/plain']);
// Staged so the qa sweep can check every one of them with our own reader
// before any of it is uploaded.
writeFileSync(join(oneDir, stagedFileName(name)), text);
}
// Unit counts, for the symbols that have more than one.
//
// The chooser tree needs them before anything is fetched: KiCad builds a
// symbol's unit rows when it builds the node (LIB_TREE_NODE_ITEM::Update ->
// AddUnit), so a multi-unit part shows its expander arrow immediately. Ours
// could only learn the count by downloading the symbol, so the arrow appeared
// after the row was clicked. Carried like `power`: only the entries that need
// it, so a library of ordinary parts adds nothing.
const units = {};
for (const name of names) {
const n = unitCountOf(blockByName.get(name) ?? '');
if (n > 1) units[name] = n;
}
symIndex.push({
name: lib,
count: names.length,
symbols: names,
// Omitted entirely when a library has none, keeping the index small.
...(power.length ? { power } : {}),
...(Object.keys(units).length ? { units } : {}),
});
// stage the merged lib so the qa sweep can validate it with our engines
mkdirSync(join(ROOT, 'tools/libraries/out/symbols'), { recursive: true });
writeFileSync(join(ROOT, 'tools/libraries/out/symbols', `${lib}.kicad_sym`), merged);
}
console.log(
`symbols: ${symIndex.length} libraries, ${symIndex.reduce((n, l) => n + l.count, 0)} symbols`,
);
// --- footprints: verbatim files ------------------------------------------------
const fpEntries = [];
const fpIndex = [];
const pretties = readdirSync(FP_SRC).filter((d) => d.endsWith('.pretty'));
for (const dir of pretties.sort()) {
const lib = dir.replace(/\.pretty$/, '');
const mods = readdirSync(join(FP_SRC, dir))
.filter((f) => f.endsWith('.kicad_mod'))
.sort();
// FOOTPRINT_INFO's three cached fields, computed here rather than in the
// browser: the alternative is downloading and parsing every candidate to
// answer "how many pads" / "does it mention SMD", which is what the Assign
// Footprints dialog used to do and is far too much to run behind a filter
// box. See tools/libraries/fp_index.mjs.
const pads = [];
const descr = [];
const tags = [];
for (const f of mods) {
const bytes = readFileSync(join(FP_SRC, dir, f));
fpEntries.push([`footprints/${dir}/${f}`, bytes, 'text/plain']);
const info = footprintIndexInfo(bytes.toString('utf8'));
pads.push(info.pads);
descr.push(info.descr);
tags.push(info.tags);
}
fpIndex.push({
name: lib,
footprints: mods.map((f) => f.replace(/\.kicad_mod$/, '')),
pads,
descr,
tags,
});
}
console.log(`footprints: ${fpIndex.length} libraries, ${fpEntries.length} footprints`);
if (process.env.STAGE_ONLY) {
console.log('STAGE_ONLY set, merged libraries staged, skipping upload.');
process.exit(0);
}
// --- upload ---------------------------------------------------------------------
const all = [...symEntries, ...fpEntries];
const totalMB = all.reduce((n, [, b]) => n + b.length, 0) / 1e6;
console.log(`uploading ${all.length} objects, ${totalMB.toFixed(0)} MB…`);
await uploadAll(all, {
onProgress: (d, t) => {
if (d % 500 === 0 || d === t) console.log(`${d}/${t}`);
},
});
await putIndex('symbols/index.json', symIndex);
await putIndex('footprints/index.json', fpIndex);
console.log('uploaded manifests');
console.log('DONE');