Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .claude/rules/testing/built-version-ci.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ For a one-line YAML edit, the invariants below and the document map are enough.

Invariants when editing:

- Playwright and ff-matrix legs in `test-smokes-built.yml` must check both the source mode and `github.event.inputs.buckets == ''`.
- Playwright and ff-matrix legs in `test-smokes-built.yml` must check both `needs.resolve-mode.outputs.mode` and `github.event.inputs.buckets == ''`.
- `test-ff-matrix.yml` owns the ff-matrix bucket glob.
- Scheduler jobs in `test-smokes-built.yml` set per-leg OS scope through their `runners:` inputs.
- Keep the per-call suffix in `test-ff-matrix.yml`'s concurrency group so sibling calls cannot cancel one another.
100 changes: 71 additions & 29 deletions .github/workflows/test-smokes-built.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,46 @@ permissions:
contents: read

jobs:
# Single source of truth for the source mode; every job below gates on its output.
# The same expression appears in `run-name` above, where job outputs are not in scope.
resolve-mode:
name: Resolve source mode
runs-on: ubuntu-latest
permissions: {}
outputs:
mode: ${{ steps.m.outputs.mode }}
steps:
- name: Resolve mode
id: m
shell: bash
env:
MODE_INPUT: ${{ (github.event_name == 'workflow_run' && 'nightly') || github.event.inputs.source || 'build' }}
UPSTREAM_RUN_ID: ${{ github.event.workflow_run.id }}
UPSTREAM_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
run: |
case "$MODE_INPUT" in
build | nightly | release) ;;
*)
echo "::error::Unexpected source mode '${MODE_INPUT}'. Expected build, nightly, or release."
exit 1
;;
esac
# Records which upstream run triggered this one; the API exposes no such pointer.
# The marker token is assembled from a variable so the copy of this script that
# GitHub prints in the step's "Run" group cannot itself satisfy the pattern the
# post-merge trigger reader matches on - the same reason `grep '[f]oo'` does not
# match its own command line.
marker="QUARTO-TRIGGER"
printf '%s event=%s upstream-run-id=%s upstream-conclusion=%s\n' \
"$marker" "$GITHUB_EVENT_NAME" "${UPSTREAM_RUN_ID:-none}" "${UPSTREAM_CONCLUSION:-none}"
echo "Resolved source mode: $MODE_INPUT"
echo "mode=$MODE_INPUT" >> "$GITHUB_OUTPUT"

# Build the selected ref (the dispatch default).
build-artifact:
name: Build quarto dist (linux-amd64)
if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build'
if: needs.resolve-mode.outputs.mode == 'build'
needs: [resolve-mode]
runs-on: ubuntu-latest
outputs:
sha: ${{ steps.rec.outputs.sha }}
Expand Down Expand Up @@ -72,8 +108,8 @@ jobs:

run-smokes-artifact:
name: Smoke tests against built artifact
if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build'
needs: [build-artifact]
if: needs.resolve-mode.outputs.mode == 'build'
needs: [resolve-mode, build-artifact]
uses: ./.github/workflows/test-smokes.yml
with:
ref: ${{ needs.build-artifact.outputs.sha }}
Expand All @@ -85,9 +121,9 @@ jobs:
run-playwright-artifact:
name: Playwright tests against built artifact
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build'
needs.resolve-mode.outputs.mode == 'build'
&& github.event.inputs.buckets == ''
needs: [build-artifact]
needs: [resolve-mode, build-artifact]
uses: ./.github/workflows/test-smokes.yml
with:
ref: ${{ needs.build-artifact.outputs.sha }}
Expand All @@ -99,9 +135,9 @@ jobs:
run-ff-matrix-artifact:
name: Feature-format matrix against built artifact
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build'
needs.resolve-mode.outputs.mode == 'build'
&& github.event.inputs.buckets == ''
needs: [build-artifact]
needs: [resolve-mode, build-artifact]
uses: ./.github/workflows/test-ff-matrix.yml
with:
ref: ${{ needs.build-artifact.outputs.sha }}
Expand All @@ -112,7 +148,8 @@ jobs:
# Test an existing published release.
resolve-release:
name: Resolve release version
if: github.event.inputs.source == 'release'
if: needs.resolve-mode.outputs.mode == 'release'
needs: [resolve-mode]
runs-on: ubuntu-latest
outputs:
version: ${{ steps.r.outputs.version }}
Expand Down Expand Up @@ -161,8 +198,8 @@ jobs:

run-smokes-release:
name: Smoke tests against published release
if: github.event.inputs.source == 'release'
needs: [resolve-release]
if: needs.resolve-mode.outputs.mode == 'release'
needs: [resolve-mode, resolve-release]
uses: ./.github/workflows/test-smokes.yml
with:
ref: refs/tags/v${{ needs.resolve-release.outputs.version }}
Expand All @@ -173,8 +210,10 @@ jobs:

run-playwright-release:
name: Playwright tests against published release
if: github.event.inputs.source == 'release' && github.event.inputs.buckets == ''
needs: [resolve-release]
if: >-
needs.resolve-mode.outputs.mode == 'release'
&& github.event.inputs.buckets == ''
needs: [resolve-mode, resolve-release]
uses: ./.github/workflows/test-smokes.yml
with:
ref: refs/tags/v${{ needs.resolve-release.outputs.version }}
Expand All @@ -185,8 +224,10 @@ jobs:

run-ff-matrix-release:
name: Feature-format matrix against published release
if: github.event.inputs.source == 'release' && github.event.inputs.buckets == ''
needs: [resolve-release]
if: >-
needs.resolve-mode.outputs.mode == 'release'
&& github.event.inputs.buckets == ''
needs: [resolve-mode, resolve-release]
uses: ./.github/workflows/test-ff-matrix.yml
with:
ref: refs/tags/v${{ needs.resolve-release.outputs.version }}
Expand All @@ -199,8 +240,9 @@ jobs:
name: Resolve create-release run
# Only successful workflow_run builds are testable.
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly'
needs.resolve-mode.outputs.mode == 'nightly'
&& (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success')
needs: [resolve-mode]
runs-on: ubuntu-latest
outputs:
run-id: ${{ steps.r.outputs.run-id }}
Expand Down Expand Up @@ -276,9 +318,9 @@ jobs:
run-smokes-nightly-linux:
name: Smoke tests against nightly build (linux)
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly'
needs.resolve-mode.outputs.mode == 'nightly'
&& needs.resolve-nightly.outputs.has-linux == 'true'
needs: [resolve-nightly]
needs: [resolve-mode, resolve-nightly]
uses: ./.github/workflows/test-smokes.yml
with:
# Publish runs add only a version/changelog commit after this SHA.
Expand All @@ -292,9 +334,9 @@ jobs:
run-smokes-nightly-windows:
name: Smoke tests against nightly build (windows)
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly'
needs.resolve-mode.outputs.mode == 'nightly'
&& needs.resolve-nightly.outputs.has-windows == 'true'
needs: [resolve-nightly]
needs: [resolve-mode, resolve-nightly]
uses: ./.github/workflows/test-smokes.yml
with:
ref: ${{ needs.resolve-nightly.outputs.sha }}
Expand All @@ -308,9 +350,9 @@ jobs:
run-smokes-nightly-mac:
name: Smoke tests against nightly build (macOS)
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly'
needs.resolve-mode.outputs.mode == 'nightly'
&& needs.resolve-nightly.outputs.has-mac == 'true'
needs: [resolve-nightly]
needs: [resolve-mode, resolve-nightly]
uses: ./.github/workflows/test-smokes.yml
with:
ref: ${{ needs.resolve-nightly.outputs.sha }}
Expand All @@ -325,10 +367,10 @@ jobs:
run-playwright-nightly-linux:
name: Playwright tests against nightly build (linux)
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly'
needs.resolve-mode.outputs.mode == 'nightly'
&& needs.resolve-nightly.outputs.has-linux == 'true'
&& github.event.inputs.buckets == ''
needs: [resolve-nightly]
needs: [resolve-mode, resolve-nightly]
uses: ./.github/workflows/test-smokes.yml
with:
ref: ${{ needs.resolve-nightly.outputs.sha }}
Expand All @@ -341,10 +383,10 @@ jobs:
run-playwright-nightly-mac:
name: Playwright tests against nightly build (macOS)
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly'
needs.resolve-mode.outputs.mode == 'nightly'
&& needs.resolve-nightly.outputs.has-mac == 'true'
&& github.event.inputs.buckets == ''
needs: [resolve-nightly]
needs: [resolve-mode, resolve-nightly]
uses: ./.github/workflows/test-smokes.yml
with:
ref: ${{ needs.resolve-nightly.outputs.sha }}
Expand All @@ -357,10 +399,10 @@ jobs:
run-ff-matrix-nightly-linux:
name: Feature-format matrix against nightly build (linux)
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly'
needs.resolve-mode.outputs.mode == 'nightly'
&& needs.resolve-nightly.outputs.has-linux == 'true'
&& github.event.inputs.buckets == ''
needs: [resolve-nightly]
needs: [resolve-mode, resolve-nightly]
uses: ./.github/workflows/test-ff-matrix.yml
with:
ref: ${{ needs.resolve-nightly.outputs.sha }}
Expand All @@ -372,10 +414,10 @@ jobs:
run-ff-matrix-nightly-windows:
name: Feature-format matrix against nightly build (windows)
if: >-
(github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly'
needs.resolve-mode.outputs.mode == 'nightly'
&& needs.resolve-nightly.outputs.has-windows == 'true'
&& github.event.inputs.buckets == ''
needs: [resolve-nightly]
needs: [resolve-mode, resolve-nightly]
uses: ./.github/workflows/test-ff-matrix.yml
with:
ref: ${{ needs.resolve-nightly.outputs.sha }}
Expand Down
2 changes: 1 addition & 1 deletion llm-docs/built-version-testing-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ In practice:

## Built-mode test legs (scheduler layout)

`test-smokes-built.yml` = the mode **resolvers** (build-artifact / resolve-nightly / resolve-release, unchanged) + a **scheduler**: per-leg caller jobs fanning out to the reusable workflows.
`test-smokes-built.yml` = a `resolve-mode` job computing the source mode once (`needs.resolve-mode.outputs.mode`, gating all other jobs) + the mode **resolvers** (build-artifact / resolve-nightly / resolve-release) + a **scheduler**: per-leg caller jobs fanning out to the reusable workflows.
Each source mode schedules three independent legs:

| leg | goes through | bucket | OS scope |
Expand Down
Loading