From 4ebaca97e726f0d897eb8c3682e8e7d3967c4b50 Mon Sep 17 00:00:00 2001 From: bherbruck Date: Sun, 18 Oct 2026 08:11:58 -0400 Subject: [PATCH] CI: run every robustness golden on pull requests that change geometry; spread fork PRs over both runner pools (#2492) #2278 dropped the golden sample from pull requests, and two geometry PRs then broke main together. Pull requests that touch the kernel, engine, doc, sketch or geom crates now run every seed, split over the shards, with the command to re-record in the failure output. Fork PRs whose number ends in 7, 8 or 9 stay on the untrusted pool; the rest build and test on GitHub's runners, which have free slots. Co-authored-by: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 37 ++++++++++++++++++++++++++++++++----- CONTRIBUTING.md | 4 ++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d67ec711..68a11b8e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,8 @@ # jobs, so they never wait behind a build), and the repository checks, Clippy, the doc tests, # the reference tests and all four test shards on the self-hosted runners (`self-hosted-fast`, warm # build directories via scripts/ci/warm-target.sh). Fork PRs' Linux jobs run on the isolated, ephemeral -# `self-hosted-untrusted` runners (no warm caches); everything else runs on GitHub's runners. +# `self-hosted-untrusted` runners (no warm caches) when the PR number ends in 7, 8 or 9 (the rest of +# the fork PRs, and everything else, run on GitHub's runners, which have far more free slots). # The test archive is always built on GitHub's runners: the self-hosted machine has a newer # glibc, so its binaries wouldn't start on ubuntu-24.04, while the archive runs there unchanged # (same /home/runner/work checkout path). Fork pull requests, the nightly run and Run workflow @@ -80,6 +81,7 @@ jobs: outputs: native: ${{ steps.paths.outputs.native }} kernel: ${{ steps.paths.outputs.kernel }} + goldens: ${{ steps.paths.outputs.goldens }} windows: ${{ steps.paths.outputs.windows }} full: ${{ steps.paths.outputs.full }} packages: ${{ steps.paths.outputs.packages }} @@ -104,6 +106,7 @@ jobs: run: | native=false kernel=false + goldens=false windows=false full=true packages= @@ -129,6 +132,10 @@ jobs: git diff --no-renames --name-only -z "$BASE_SHA...$HEAD_SHA" > "$RUNNER_TEMP/ci-paths" while IFS= read -r -d '' path; do case "$path" in crates/kernel/*|vendor/truck-*) kernel=true ;; esac + # What can change a robustness golden: the kernel and the crates a design is built with. + case "$path" in + crates/kernel/*|crates/engine/*|crates/doc/*|crates/sketch/*|crates/geom/*|vendor/truck-*) goldens=true ;; + esac # What the Windows D3D12 jobs exercise: the GPU crate and its shaders, the app and # its startup, the viewport's wgpu glue, the vendored egui-wgpu, their scripts. case "$path" in @@ -155,6 +162,10 @@ jobs: packages=$(sed -n 's/^packages=//p' "$RUNNER_TEMP/ci-affected") # Code changed but nothing was selected: be safe and test everything. if [ "$native" = true ] && [ "$full" != true ] && [ -z "$packages" ]; then full=true; fi + # The goldens are solidcraft-engine tests: its tests must be in the archive. + if [ "$goldens" = true ] && [ "$full" != true ]; then + case " $packages " in *" solidcraft-engine "*) ;; *) packages="$packages solidcraft-engine" ;; esac + fi fi # The reference tests replay recipes through solidcraft-cli: needed when it, or anything # it depends on, is selected. @@ -168,6 +179,7 @@ jobs: { echo "native=$native" echo "kernel=$kernel" + echo "goldens=$goldens" echo "windows=$windows" echo "oracle=$oracle" echo "full=$full" @@ -249,9 +261,9 @@ jobs: name: Build tests needs: changes if: needs.changes.outputs.native == 'true' - runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'self-hosted-untrusted' || 'ubuntu-24.04' }} + runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 'self-hosted-untrusted' || 'ubuntu-24.04' }} # Fork builds compile inside gVisor on shared boxes; give them room to finish and upload. - timeout-minutes: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 90 || 45 }} + timeout-minutes: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 90 || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 60 || 45 }} env: CARGO_INCREMENTAL: "0" RUSTC_WRAPPER: sccache @@ -295,7 +307,7 @@ jobs: tests: name: Native Linux tests (${{ matrix.shard }}/4) needs: [changes, build-tests] - runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'self-hosted-untrusted' || 'ubuntu-24.04') }} + runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 'self-hosted-untrusted' || 'ubuntu-24.04') }} timeout-minutes: 60 strategy: fail-fast: false @@ -339,6 +351,21 @@ jobs: # the build job's path, which is the same checkout path here. cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \ --profile ci --partition "slice:$SHARD/$SHARDS" --no-tests=pass "${all[@]}" + - name: Robustness goldens (pull requests that change geometry) + # Every seed of tests/robustness/goldens.json (each seed is a test), split over the shards; + # pushes to main and the nightly run them in the step above. + if: github.event_name == 'pull_request' && needs.changes.outputs.goldens == 'true' + env: + SHARD: ${{ matrix.shard }} + SHARDS: ${{ github.event_name == 'pull_request' && 2 || 4 }} + run: | + if ! cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \ + --profile ci --ignore-default-filter --partition "slice:$SHARD/$SHARDS" --no-tests=pass \ + -E 'package(solidcraft-engine) & test(/^tests::robustness_goldens::/)'; then + echo "::error::A robustness golden changed. If the new result is intended, re-record it in this pull request and say why:" + echo "::error::ROBUSTNESS_GOLDENS_BLESS=\"why it changed\" cargo test --profile ci -p solidcraft-engine --lib robustness_goldens:: (see docs/robustness.md)" + exit 1 + fi - name: Upload test timings if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -496,7 +523,7 @@ jobs: name: Reference tests needs: changes if: needs.changes.outputs.oracle == 'true' - runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'self-hosted-untrusted' || 'ubuntu-24.04') }} + runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 'self-hosted-untrusted' || 'ubuntu-24.04') }} timeout-minutes: 60 env: CARGO_INCREMENTAL: "0" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c0f5b755..61f005faf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,6 +138,10 @@ the most complete description of the house style. | UI scenarios | `crates/ui-egui/tests/scenarios/*.json` | part of `cargo test` | | Reference tests | `cargo xtask oracle` | `tests/oracle/` (see below) | +A geometry change that changes the robustness goldens re-records them in the same pull request: +`ROBUSTNESS_GOLDENS_BLESS="why it changed" cargo test --profile ci -p solidcraft-engine --lib robustness_goldens::` +(see [docs/robustness.md](docs/robustness.md)); CI runs every seed on pull requests that touch geometry. + **UI scenarios** drive the real app headless: start a tool, click a plane or a sketch point, press keys, then assert on the result.