CI: run every robustness golden on pull requests that change geometry; spread fork PRs over both runner pools (#2492)
#2278 dropped the golden sample from pull requests, and two geometry PRs then broke main together. Pull requests that touch the kernel, engine, doc, sketch or geom crates now run every seed, split over the shards, with the command to re-record in the failure output. Fork PRs whose number ends in 7, 8 or 9 stay on the untrusted pool; the rest build and test on GitHub's runners, which have free slots. Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,7 +21,8 @@
|
||||
# jobs, so they never wait behind a build), and the repository checks, Clippy, the doc tests,
|
||||
# the reference tests and all four test shards on the self-hosted runners (`self-hosted-fast`, warm
|
||||
# build directories via scripts/ci/warm-target.sh). Fork PRs' Linux jobs run on the isolated, ephemeral
|
||||
# `self-hosted-untrusted` runners (no warm caches); everything else runs on GitHub's runners.
|
||||
# `self-hosted-untrusted` runners (no warm caches) when the PR number ends in 7, 8 or 9 (the rest of
|
||||
# the fork PRs, and everything else, run on GitHub's runners, which have far more free slots).
|
||||
# The test archive is always built on GitHub's runners: the self-hosted machine has a newer
|
||||
# glibc, so its binaries wouldn't start on ubuntu-24.04, while the archive runs there unchanged
|
||||
# (same /home/runner/work checkout path). Fork pull requests, the nightly run and Run workflow
|
||||
@@ -80,6 +81,7 @@ jobs:
|
||||
outputs:
|
||||
native: ${{ steps.paths.outputs.native }}
|
||||
kernel: ${{ steps.paths.outputs.kernel }}
|
||||
goldens: ${{ steps.paths.outputs.goldens }}
|
||||
windows: ${{ steps.paths.outputs.windows }}
|
||||
full: ${{ steps.paths.outputs.full }}
|
||||
packages: ${{ steps.paths.outputs.packages }}
|
||||
@@ -104,6 +106,7 @@ jobs:
|
||||
run: |
|
||||
native=false
|
||||
kernel=false
|
||||
goldens=false
|
||||
windows=false
|
||||
full=true
|
||||
packages=
|
||||
@@ -129,6 +132,10 @@ jobs:
|
||||
git diff --no-renames --name-only -z "$BASE_SHA...$HEAD_SHA" > "$RUNNER_TEMP/ci-paths"
|
||||
while IFS= read -r -d '' path; do
|
||||
case "$path" in crates/kernel/*|vendor/truck-*) kernel=true ;; esac
|
||||
# What can change a robustness golden: the kernel and the crates a design is built with.
|
||||
case "$path" in
|
||||
crates/kernel/*|crates/engine/*|crates/doc/*|crates/sketch/*|crates/geom/*|vendor/truck-*) goldens=true ;;
|
||||
esac
|
||||
# What the Windows D3D12 jobs exercise: the GPU crate and its shaders, the app and
|
||||
# its startup, the viewport's wgpu glue, the vendored egui-wgpu, their scripts.
|
||||
case "$path" in
|
||||
@@ -155,6 +162,10 @@ jobs:
|
||||
packages=$(sed -n 's/^packages=//p' "$RUNNER_TEMP/ci-affected")
|
||||
# Code changed but nothing was selected: be safe and test everything.
|
||||
if [ "$native" = true ] && [ "$full" != true ] && [ -z "$packages" ]; then full=true; fi
|
||||
# The goldens are solidcraft-engine tests: its tests must be in the archive.
|
||||
if [ "$goldens" = true ] && [ "$full" != true ]; then
|
||||
case " $packages " in *" solidcraft-engine "*) ;; *) packages="$packages solidcraft-engine" ;; esac
|
||||
fi
|
||||
fi
|
||||
# The reference tests replay recipes through solidcraft-cli: needed when it, or anything
|
||||
# it depends on, is selected.
|
||||
@@ -168,6 +179,7 @@ jobs:
|
||||
{
|
||||
echo "native=$native"
|
||||
echo "kernel=$kernel"
|
||||
echo "goldens=$goldens"
|
||||
echo "windows=$windows"
|
||||
echo "oracle=$oracle"
|
||||
echo "full=$full"
|
||||
@@ -249,9 +261,9 @@ jobs:
|
||||
name: Build tests
|
||||
needs: changes
|
||||
if: needs.changes.outputs.native == 'true'
|
||||
runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'self-hosted-untrusted' || 'ubuntu-24.04' }}
|
||||
runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 'self-hosted-untrusted' || 'ubuntu-24.04' }}
|
||||
# Fork builds compile inside gVisor on shared boxes; give them room to finish and upload.
|
||||
timeout-minutes: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 90 || 45 }}
|
||||
timeout-minutes: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 90 || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 60 || 45 }}
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
RUSTC_WRAPPER: sccache
|
||||
@@ -295,7 +307,7 @@ jobs:
|
||||
tests:
|
||||
name: Native Linux tests (${{ matrix.shard }}/4)
|
||||
needs: [changes, build-tests]
|
||||
runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'self-hosted-untrusted' || 'ubuntu-24.04') }}
|
||||
runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 'self-hosted-untrusted' || 'ubuntu-24.04') }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -339,6 +351,21 @@ jobs:
|
||||
# the build job's path, which is the same checkout path here.
|
||||
cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \
|
||||
--profile ci --partition "slice:$SHARD/$SHARDS" --no-tests=pass "${all[@]}"
|
||||
- name: Robustness goldens (pull requests that change geometry)
|
||||
# Every seed of tests/robustness/goldens.json (each seed is a test), split over the shards;
|
||||
# pushes to main and the nightly run them in the step above.
|
||||
if: github.event_name == 'pull_request' && needs.changes.outputs.goldens == 'true'
|
||||
env:
|
||||
SHARD: ${{ matrix.shard }}
|
||||
SHARDS: ${{ github.event_name == 'pull_request' && 2 || 4 }}
|
||||
run: |
|
||||
if ! cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \
|
||||
--profile ci --ignore-default-filter --partition "slice:$SHARD/$SHARDS" --no-tests=pass \
|
||||
-E 'package(solidcraft-engine) & test(/^tests::robustness_goldens::/)'; then
|
||||
echo "::error::A robustness golden changed. If the new result is intended, re-record it in this pull request and say why:"
|
||||
echo "::error::ROBUSTNESS_GOLDENS_BLESS=\"why it changed\" cargo test --profile ci -p solidcraft-engine --lib robustness_goldens:: (see docs/robustness.md)"
|
||||
exit 1
|
||||
fi
|
||||
- name: Upload test timings
|
||||
if: always()
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
@@ -496,7 +523,7 @@ jobs:
|
||||
name: Reference tests
|
||||
needs: changes
|
||||
if: needs.changes.outputs.oracle == 'true'
|
||||
runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'self-hosted-untrusted' || 'ubuntu-24.04') }}
|
||||
runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 'self-hosted-untrusted' || 'ubuntu-24.04') }}
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
|
||||
Reference in New Issue
Block a user