diff --git a/.github/workflows/_detect-changes.yml b/.github/workflows/_detect-changes.yml index 80354c1..d02d378 100644 --- a/.github/workflows/_detect-changes.yml +++ b/.github/workflows/_detect-changes.yml @@ -9,6 +9,14 @@ name: Detect Changes (reusable) # # Pairs with _verified.yml rather than competing with it: the marker skips a run # whose bytes already passed, this one skips jobs the change cannot affect. +# +# `fingerprint: true` swaps the diff for a per-area content check. Each filter +# gets a fingerprint over the blobs its patterns match in the tree under test, +# and an area runs unless a green run already recorded exactly that fingerprint +# as a `ci-area/` commit status (written by _record-area-fingerprints.yml). +# Not a diff against the previous run: a red or cancelled predecessor does not +# count, and a revert, a retry, a release PR or the push that follows a merged +# PR all land on a fingerprint that already passed. on: workflow_call: inputs: @@ -32,14 +40,35 @@ on: required: false type: boolean default: false + fingerprint: + description: >- + Decide per area by content fingerprint instead of the diff: an area is + skipped only when a green run already recorded its exact fingerprint. + Needs `contents: read` and `statuses: read` on the calling job (without + the latter every area runs) and `pull-requests: read` for a squash merge + to find what its PR recorded. Pair with _record-area-fingerprints.yml. + required: false + type: boolean + default: false outputs: changes: description: JSON array of the filter names that matched. Gate with `contains(fromJSON(...), 'php')`. - value: ${{ jobs.detect.outputs.changes }} + value: ${{ jobs.fingerprint.outputs.changes || jobs.detect.outputs.changes }} + fingerprints: + description: >- + With `fingerprint: true`, a JSON object of area name → fingerprint for + every area that has one. Hand it to _record-area-fingerprints.yml. + value: ${{ jobs.fingerprint.outputs.fingerprints || '{}' }} + verified: + description: >- + With `fingerprint: true`, a JSON array of the areas skipped because + their fingerprint already passed. + value: ${{ jobs.fingerprint.outputs.verified || '[]' }} jobs: detect: name: Detect Changes + if: ${{ !inputs.fingerprint }} runs-on: ${{ inputs.runner }} timeout-minutes: 5 permissions: @@ -78,3 +107,221 @@ jobs: with: base: ${{ github.event_name == 'push' && github.ref || '' }} filters: ${{ inputs.filters }} + + fingerprint: + name: Detect Changes by Fingerprint + if: inputs.fingerprint + runs-on: ${{ inputs.runner }} + timeout-minutes: 5 + # No `permissions:` block on purpose. A block here would be validated at + # startup for EVERY caller, fingerprint or not, and `statuses: read` would + # then break each repo that never granted it. Without one the job inherits + # exactly what the calling job grants: `contents: read` to fetch the tree, + # `statuses: read` to see earlier green areas, `pull-requests: read` to + # follow a squash merge back to its PR. + outputs: + changes: ${{ steps.decide.outputs.changes }} + fingerprints: ${{ steps.compute.outputs.fingerprints }} + verified: ${{ steps.decide.outputs.verified }} + steps: + # Trees only, no blobs and no checkout: `read-tree` fills an index from + # tree objects, and `ls-files -s` lists the blob id of every path a pattern + # matches. That listing is the area's content; its hash is the fingerprint. + # + # On a pull request GITHUB_SHA is the MERGE commit, so the fingerprint names + # what would land — which is why the push after the merge finds it again. + # + # Only the dorny filter subset whose meaning git's `:(glob)` pathspec shares + # is fingerprinted: plain, quoted or unquoted list items. Negations, status + # prefixes, anchors, braces and extglobs would match differently or not at + # all, and a fingerprint that misses files would skip an area that changed. + # Such an area gets no fingerprint and always runs. + - name: Compute the area fingerprints + id: compute + env: + FILTERS: ${{ inputs.filters }} + TOKEN: ${{ github.token }} + SERVER_URL: ${{ github.server_url }} + REPOSITORY: ${{ github.repository }} + SHA: ${{ github.sha }} + run: | + set -uo pipefail + + names=$(printf '%s\n' "$FILTERS" | grep -oE '^[a-zA-Z0-9_-]+:' | tr -d ':' | jq -R . | jq -sc .) + echo "names=$names" >> "$GITHUB_OUTPUT" + + # Fail open. No tree means no fingerprint, and no fingerprint means the + # area runs — a redundant run, never a skipped one. + give_up() { + echo "::warning::$1 — every area runs." + echo 'fingerprints={}' >> "$GITHUB_OUTPUT" + exit 0 + } + + work="${RUNNER_TEMP:-/tmp}/area-fingerprints" + rm -rf "$work" + git init -q "$work" || give_up "Could not create a scratch repository" + auth=$(printf 'x-access-token:%s' "$TOKEN" | base64 | tr -d '\n') + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraheader GIT_CONFIG_VALUE_0="AUTHORIZATION: basic $auth" \ + git -C "$work" fetch -q --depth=1 --filter=blob:none --no-tags "$SERVER_URL/$REPOSITORY" "$SHA" \ + || give_up "Could not fetch the tree of $SHA" + git -C "$work" read-tree "$SHA" || give_up "Could not read the tree of $SHA" + + parsed="$work.filters" + printf '%s\n' "$FILTERS" | awk ' + /^[[:space:]]*(#.*)?$/ { next } + /^[A-Za-z0-9_-]+:/ { + name = $0; sub(/:.*/, "", name) + rest = $0; sub(/^[^:]*:[[:space:]]*/, "", rest) + if (rest !~ /^(#.*)?$/) print "X\t" name + next + } + name == "" { next } + /^[[:space:]]+-[[:space:]]+/ { + p = $0; sub(/^[[:space:]]+-[[:space:]]+/, "", p) + if (p ~ /^\047/) { + if (p !~ /^\047[^\047]*\047[[:space:]]*(#.*)?$/) { print "X\t" name; next } + sub(/^\047/, "", p); sub(/\047.*$/, "", p) + } else if (p ~ /^"/) { + if (p !~ /^"[^"\\]*"[[:space:]]*(#.*)?$/) { print "X\t" name; next } + sub(/^"/, "", p); sub(/".*$/, "", p) + } else { + sub(/[[:space:]]+#.*$/, "", p); sub(/[[:space:]]+$/, "", p) + if (p ~ /^[*&|>%@`]/ || p ~ /:([[:space:]]|$)/) { print "X\t" name; next } + } + if (p == "" || p ~ /^!/ || p ~ /[{}()\\]/) { print "X\t" name; next } + print "P\t" name "\t" p + next + } + { print "X\t" name } + ' > "$parsed" || give_up "Could not parse the filters" + + fingerprints='{}' + while IFS= read -r name; do + mapfile -t patterns < <(awk -F'\t' -v n="$name" '$1 == "P" && $2 == n { print $3 }' "$parsed") + if grep -qxF "X $name" "$parsed" || [ "${#patterns[@]}" -eq 0 ]; then + echo "::warning::Area '$name' uses filter syntax a fingerprint cannot reproduce (negation, status prefix, anchor, brace or extglob) — it runs every time." + continue + fi + specs=() + for pattern in "${patterns[@]}"; do specs+=(":(glob)$pattern"); done + listing=$(git -C "$work" ls-files -s -- "${specs[@]}") || give_up "Could not list the files of area '$name'" + # The patterns are part of the input: widening a filter must not + # inherit the verdict the narrower one earned. + fingerprint=$({ printf 'ci-area/v1\n'; printf '%s\n' "${patterns[@]}"; printf '%s\n' "$listing"; } | sha256sum | cut -d' ' -f1) + fingerprints=$(jq -c --arg n "$name" --arg f "$fingerprint" '. + {($n): $f}' <<< "$fingerprints") + done < <(jq -r '.[]' <<< "$names") + + echo "fingerprints=$fingerprints" >> "$GITHUB_OUTPUT" + echo "Fingerprints: $fingerprints" + + # One GraphQL call collects the latest status contexts of the last 100 + # commits of three histories, plus the commits of up to five pull requests + # associated with this one: + # - this commit (the PR head on a pull request): its own retries, the PR's + # earlier pushes, and its branch's ancestry; + # - the base branch (the pushed branch on a push), for newer commits; + # - the default branch, where a PR into `dev` meets what `main` passed; + # - the associated pull requests, because a squash or rebase merge leaves + # the PR's commits — and the statuses on them — out of every history. + # The status is only storage: the match is on context plus fingerprint, so + # whichever commit carries it is irrelevant. + - name: Look up fingerprints that already passed + id: lookup + if: ${{ !inputs.force && steps.compute.outputs.fingerprints != '{}' }} + env: + TOKEN: ${{ github.token }} + API_URL: ${{ github.api_url }} + GRAPHQL_URL: ${{ github.graphql_url }} + REPOSITORY: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE: ${{ github.base_ref || github.ref_name }} + run: | + set -uo pipefail + + give_up() { + echo "::warning::$1 — every area runs." + echo 'green=[]' >> "$GITHUB_OUTPUT" + exit 0 + } + + # A token without `statuses: read` does not make GraphQL fail — it + # answers with no statuses, which reads as "nothing passed yet" and + # would hide the missing grant forever. The REST endpoint refuses + # outright, so it turns that into a warning someone can act on. + curl -fsS --max-time 30 -o /dev/null \ + -H "Authorization: Bearer $TOKEN" -H 'Accept: application/vnd.github+json' \ + "$API_URL/repos/$REPOSITORY/commits/$SHA/status" \ + || give_up "Cannot read commit statuses — grant 'statuses: read' to the job calling _detect-changes.yml" + + query=' + query($owner: String!, $name: String!, $sha: GitObjectID!, $base: String!, $n: Int!) { + repository(owner: $owner, name: $name) { + defaultBranchRef { target { ...History } } + base: ref(qualifiedName: $base) { target { ...History } } + object(oid: $sha) { + ...History + ... on Commit { + associatedPullRequests(first: 5) { nodes { commits(last: $n) { nodes { commit { ...Status } } } } } + } + } + } + } + fragment History on GitObject { ... on Commit { history(first: $n) { nodes { ...Status } } } } + fragment Status on Commit { status { contexts { context state description } } }' + + body=$(jq -nc --arg q "$query" --arg owner "${REPOSITORY%%/*}" --arg name "${REPOSITORY#*/}" \ + --arg sha "$SHA" --arg base "refs/heads/$BASE" \ + '{query: $q, variables: {owner: $owner, name: $name, sha: $sha, base: $base, n: 100}}') + answer=$(curl -fsS --max-time 60 -X POST \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + --data "$body" "$GRAPHQL_URL") || give_up "The status lookup failed" + + if ! jq -e '.data.repository' <<< "$answer" > /dev/null 2>&1; then + give_up "The status lookup returned no repository: $(jq -c '.errors' <<< "$answer" 2> /dev/null)" + fi + # Partial answers are still answers. Without `pull-requests: read` the + # associated pull requests come back as an error beside the histories; + # the histories still count, only a squash merge loses its PR's records. + if jq -e '.errors' <<< "$answer" > /dev/null 2>&1; then + echo "::warning::Part of the status lookup failed, the rest still counts: $(jq -c '[.errors[].message]' <<< "$answer")" + fi + + green=$(jq -c '[.. | objects | select(has("context") and has("state")) + | select(.state == "SUCCESS" and (.context | startswith("ci-area/"))) + | {context, description}] | unique' <<< "$answer") || give_up "Could not read the status answer" + echo "green=$green" >> "$GITHUB_OUTPUT" + echo "Green area records found: $(jq length <<< "$green")" + + # An area runs when it is forced, has no fingerprint, or its fingerprint + # has no green record. Nothing else — no diff, no predecessor's verdict. + - name: Decide which areas run + id: decide + env: + FORCE: ${{ inputs.force }} + NAMES: ${{ steps.compute.outputs.names }} + FINGERPRINTS: ${{ steps.compute.outputs.fingerprints }} + GREEN: ${{ steps.lookup.outputs.green }} + run: | + set -euo pipefail + fps=$FINGERPRINTS + [ -n "$fps" ] || fps='{}' + decision=$(jq -nc --argjson names "${NAMES:-[]}" --argjson fps "$fps" \ + --argjson green "${GREEN:-[]}" --arg force "$FORCE" ' + ($green | map("\(.context)=\(.description)")) as $passed + | [$names[] | . as $n | { + name: $n, + fingerprint: $fps[$n], + verified: ($force != "true" and $fps[$n] != null + and any($passed[]; . == "ci-area/\($n)=\($fps[$n])")) + }]') + changes=$(jq -c '[.[] | select(.verified | not) | .name]' <<< "$decision") + verified=$(jq -c '[.[] | select(.verified) | .name]' <<< "$decision") + echo "changes=$changes" >> "$GITHUB_OUTPUT" + echo "verified=$verified" >> "$GITHUB_OUTPUT" + echo "Run: $changes — already green: $verified" + { + echo '| Area | Fingerprint | Decision |' + echo '| :--- | :---------- | :------- |' + jq -r '.[] | "| `\(.name)` | `\(.fingerprint // "none" | .[0:12])` | \(if .verified then "already green" else "runs" end) |"' <<< "$decision" + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" diff --git a/.github/workflows/_record-area-fingerprints.yml b/.github/workflows/_record-area-fingerprints.yml new file mode 100644 index 0000000..6b90582 --- /dev/null +++ b/.github/workflows/_record-area-fingerprints.yml @@ -0,0 +1,115 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +name: Record Area Fingerprints (reusable) + +# The other half of `_detect-changes.yml` with `fingerprint: true`. The caller +# runs it after its Gate went green and hands over the `fingerprints` output; +# each named area becomes a `ci-area/` commit status whose description is +# the fingerprint, which is what the next lookup matches on. +# +# A COMMIT STATUS, NOT `actions/cache` OR A FILE. The cache expires after seven +# days and is branch-scoped; a file is one more thing parallel runs overwrite. +# A status is per commit and per context, so two runs writing the same area at +# once write the same answer. +# +# THE STATUS IS A CLAIM ABOUT WHAT PASSED, so the caller decides what it covers +# through `areas`. Record an area only when every job gated on it ran green or +# was skipped because `_detect-changes.yml` reported it as already verified — +# never when it was skipped for any other reason (a draft, a release-only PR, a +# job-level `if:`). Recording the verified areas again is deliberate: it carries +# their fingerprint onto this commit, so the lookup window never loses it. +on: + workflow_call: + inputs: + runner: + description: >- + Runner label for this body's jobs. THE ONE THING A BODY CANNOT DERIVE: + nothing in a repository states which fleet it belongs to, and the three + repos that run on Blacksmith chose that deliberately rather than + inheriting it. Default keeps every other caller on GitHub-hosted. + required: false + type: string + default: ubuntu-latest + fingerprints: + description: The `fingerprints` output of _detect-changes.yml. + required: true + type: string + areas: + description: >- + JSON array of the area names this run proved green, or `*` for every + area in `fingerprints`. Names without a fingerprint are ignored. + required: true + type: string + +jobs: + record: + name: Record Green Areas + runs-on: ${{ inputs.runner }} + timeout-minutes: 5 + permissions: + statuses: write + steps: + # On a pull request the status goes onto the HEAD commit, not the merge + # commit the fingerprint was computed from: the merge commit belongs to no + # history the lookup walks, the head commit does. The fingerprint itself + # still names the merged tree, so it is the content that matches, never + # the commit carrying it. + - name: Mark the green areas on this commit + env: + TOKEN: ${{ github.token }} + API_URL: ${{ github.api_url }} + REPOSITORY: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + FINGERPRINTS: ${{ inputs.fingerprints }} + AREAS: ${{ inputs.areas }} + run: | + set -euo pipefail + + # A fork's token is read-only. Failing here would turn a green pull + # request red for a record that costs, at worst, one redundant run. + if [ "$HEAD_REPOSITORY" != "$REPOSITORY" ]; then + echo "::notice::Pull request from a fork — its token cannot write statuses, nothing recorded." + exit 0 + fi + + if ! jq -e 'type == "object"' <<< "${FINGERPRINTS:-null}" > /dev/null 2>&1; then + echo "::notice::No fingerprints — _detect-changes.yml failed open or ran without fingerprint, nothing to record." + exit 0 + fi + if [ "$AREAS" != '*' ] && ! jq -e 'type == "array" and all(.[]; type == "string")' <<< "$AREAS" > /dev/null 2>&1; then + echo "::error::'areas' must be a JSON array of area names or '*', got: $AREAS" + exit 1 + fi + + pairs=$(jq -r --arg areas "$AREAS" ' + . as $fps + | (if $areas == "*" then keys_unsorted else ($areas | fromjson) end) + | .[] | select($fps[.] != null) | "\(.)\t\($fps[.])"' <<< "$FINGERPRINTS") + + if [ -z "$pairs" ]; then + echo "::notice::None of the named areas has a fingerprint, nothing to record." + exit 0 + fi + + failed=0 + while IFS=$'\t' read -r name fingerprint; do + if ! [[ "$name" =~ ^[A-Za-z0-9_-]+$ && "$fingerprint" =~ ^[0-9a-f]{64}$ ]]; then + echo "::error::Refusing to record '$name' with fingerprint '$fingerprint'." + failed=1 + continue + fi + if jq -nc --arg context "ci-area/$name" --arg fingerprint "$fingerprint" --arg url "$RUN_URL" \ + '{state: "success", context: $context, description: $fingerprint, target_url: $url}' \ + | curl -fsS --max-time 30 -o /dev/null -X POST \ + -H "Authorization: Bearer $TOKEN" -H 'Accept: application/vnd.github+json' \ + --data @- "$API_URL/repos/$REPOSITORY/statuses/$SHA" + then + echo "::notice::Recorded ci-area/$name as green for $fingerprint." + else + echo "::error::Could not record ci-area/$name — grant 'statuses: write' to the calling job." + failed=1 + fi + done <<< "$pairs" + exit "$failed" diff --git a/AGENTS.md b/AGENTS.md index fd92f08..2e44d12 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -226,7 +226,7 @@ A body has no trigger of its own, so this repo needs a stub like anybody else. I The task list is derived from the repo's own `package.json` rather than passed in: `resolve` splits the gate script on `&&`, keeps each bare `pnpm ` call, and expands recursively, since a gate may chain another aggregate (`skills` runs `verify` → `check` → `lint && format`). That is what carries the repo-specific checks — `validate`, `check:policy`, `skills:check` — along without listing them here; a fixed candidate list would have dropped exactly those. A task taking arguments is left unexpanded, because running it standalone would change what it does. -All of it is built: `_verified.yml` / `_record-verified.yml` (the tree-hash markers from `app` — `skip` gates the caller's own jobs, so a repo takes the markers without handing over its pipeline), `_detect-changes.yml` (path filters, passed in because they describe a layout only the repo knows), `_ci-node.yml`, `_ci-laravel.yml`, `_ci-go.yml`, `_ci-tofu.yml` and `_coverage.yml` around `kirchDev/coverage-report`. +All of it is built: `_verified.yml` / `_record-verified.yml` (the tree-hash markers from `app` — `skip` gates the caller's own jobs, so a repo takes the markers without handing over its pipeline), `_detect-changes.yml` (path filters, passed in because they describe a layout only the repo knows — and, with `fingerprint: true`, a per-area content fingerprint that `_record-area-fingerprints.yml` stores as a `ci-area/` commit status after a green gate; the fingerprint job carries **no `permissions:` block** on purpose, since a block is validated at startup for every caller and `statuses: read` would break each repo that never granted it), `_ci-node.yml`, `_ci-laravel.yml`, `_ci-go.yml`, `_ci-tofu.yml` and `_coverage.yml` around `kirchDev/coverage-report`. **Deliberately NOT bodies, measured rather than assumed:** diff --git a/CLAUDE.md b/CLAUDE.md index fd92f08..2e44d12 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -226,7 +226,7 @@ A body has no trigger of its own, so this repo needs a stub like anybody else. I The task list is derived from the repo's own `package.json` rather than passed in: `resolve` splits the gate script on `&&`, keeps each bare `pnpm ` call, and expands recursively, since a gate may chain another aggregate (`skills` runs `verify` → `check` → `lint && format`). That is what carries the repo-specific checks — `validate`, `check:policy`, `skills:check` — along without listing them here; a fixed candidate list would have dropped exactly those. A task taking arguments is left unexpanded, because running it standalone would change what it does. -All of it is built: `_verified.yml` / `_record-verified.yml` (the tree-hash markers from `app` — `skip` gates the caller's own jobs, so a repo takes the markers without handing over its pipeline), `_detect-changes.yml` (path filters, passed in because they describe a layout only the repo knows), `_ci-node.yml`, `_ci-laravel.yml`, `_ci-go.yml`, `_ci-tofu.yml` and `_coverage.yml` around `kirchDev/coverage-report`. +All of it is built: `_verified.yml` / `_record-verified.yml` (the tree-hash markers from `app` — `skip` gates the caller's own jobs, so a repo takes the markers without handing over its pipeline), `_detect-changes.yml` (path filters, passed in because they describe a layout only the repo knows — and, with `fingerprint: true`, a per-area content fingerprint that `_record-area-fingerprints.yml` stores as a `ci-area/` commit status after a green gate; the fingerprint job carries **no `permissions:` block** on purpose, since a block is validated at startup for every caller and `statuses: read` would break each repo that never granted it), `_ci-node.yml`, `_ci-laravel.yml`, `_ci-go.yml`, `_ci-tofu.yml` and `_coverage.yml` around `kirchDev/coverage-report`. **Deliberately NOT bodies, measured rather than assumed:** diff --git a/README.md b/README.md index 9891650..59bf1f7 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Then delete the body that used to live in that file. Nothing else moves. List `s A **body** is prefixed with `_` and runs only via `workflow_call`. Anything without the prefix is an ordinary workflow — including this repo's own stubs. -Nineteen bodies in four groups: **branch flow** (`_promotion-pr.yml`, `_queue-branch.yml`, `_fast-forward-queue.yml`), **releasing** (`_release-please.yml`, `_publish-npm.yml`, `_publish-goreleaser.yml`), **CI** (`_ci-check.yml`, plus one body per stack — Node, Laravel, Go, OpenTofu, Rust, E2E, Lighthouse), and **reporting and short-circuits** (`_codeql.yml`, `_coverage.yml`, `_verified.yml`, `_record-verified.yml`, `_detect-changes.yml`). +Nineteen bodies in four groups: **branch flow** (`_promotion-pr.yml`, `_queue-branch.yml`, `_fast-forward-queue.yml`), **releasing** (`_release-please.yml`, `_publish-npm.yml`, `_publish-goreleaser.yml`), **CI** (`_ci-check.yml`, plus one body per stack — Node, Laravel, Go, OpenTofu, Rust, E2E, Lighthouse), and **reporting and short-circuits** (`_codeql.yml`, `_coverage.yml`, `_verified.yml`, `_record-verified.yml`, `_detect-changes.yml`, `_record-area-fingerprints.yml`).
All 19 bodies, and why ci.yml is not one of them @@ -74,6 +74,7 @@ Nineteen bodies in four groups: **branch flow** (`_promotion-pr.yml`, `_queue-br | `_verified.yml` | Has this tree already passed? — outputs `skip` | ✅ reusable | | `_record-verified.yml` | Marks a tree verified once the caller's jobs are green | ✅ reusable | | `_detect-changes.yml` | Which parts a PR touched — outputs `changes` | ✅ reusable | +| `_record-area-fingerprints.yml` | Marks areas green by content fingerprint, as commit statuses | ✅ reusable | | `_ci-node.yml` | typecheck, test, build for Node libraries | ✅ reusable | | `_ci-laravel.yml` | Pest across PHP × Laravel, PostgreSQL, PCOV, Pint | ✅ reusable | | `_ci-go.yml` | gofmt, vet, golangci-lint, build, unit + acceptance | ✅ reusable | @@ -339,6 +340,45 @@ The marker keys on the **tree hash**, so it names the content rather than the co **Which jobs the marker stands for is the caller's decision** — that is why `record-verified` takes its `if:` in the stub. Gating it on lint alone would record a tree as verified that was never tested. +#### Per area, by content fingerprint + +`_verified.yml` is all or nothing: one changed byte anywhere runs the whole suite. `_detect-changes.yml` with `fingerprint: true` makes the same decision **per area** — each filter gets a fingerprint over the blobs its patterns match, and an area is skipped only when a green run already recorded exactly that fingerprint. A red or cancelled predecessor does not count, and there is no diff against it: a revert, a retry, a release PR and the push that follows a merged PR all land on a fingerprint that already passed. + +```yaml +jobs: + changes: + permissions: + contents: read + statuses: read # without it every area runs, with a warning + pull-requests: read # lets a squash merge find what its PR recorded + uses: kirchDev/workflows/.github/workflows/_detect-changes.yml@ # v0.x.y + with: + fingerprint: true + filters: | + core: + - services/core/** + - .github/workflows/ci.yml + + # … jobs gated on contains(fromJSON(needs.changes.outputs.changes), 'core'), then gate … + + record-areas: + needs: [changes, gate] + if: needs.gate.result == 'success' && github.event.pull_request.draft != true + permissions: + statuses: write + uses: kirchDev/workflows/.github/workflows/_record-area-fingerprints.yml@ # v0.x.y + with: + fingerprints: ${{ needs.changes.outputs.fingerprints }} + areas: '*' +``` + +The record is a `ci-area/` **commit status** with the fingerprint as its description — not `actions/cache`, which expires after seven days, and not a file parallel runs would overwrite. The lookup reads the last 100 commits of this commit's history, the base branch and the default branch, plus the commits of associated pull requests, so a squash merge still finds what its PR recorded. Recording the verified areas again is what keeps them inside that window. + +- **`areas` is a claim about what passed.** Pass `'*'` only when every area job is gated on `changes` alone. An area skipped for any other reason — a draft, a release-only PR, a job-level `if:` — must be left out. +- **List the CI files in the filters.** The fingerprint covers what the patterns match and nothing else; an area whose filter omits `ci.yml` keeps its green verdict across a change to the job that checks it. +- **Only the plain filter subset is fingerprinted** — list items, quoted or not, matched through git's `:(glob)` pathspec. Negations, status prefixes, anchors, braces and extglobs would match differently, so such an area gets no fingerprint and runs every time, with a warning. +- **Fails open everywhere.** No tree, no lookup or no statuses permission means the area runs; a fork's read-only token records nothing. + ### When a body almost fits A body is taken whole or not at all — there is no way to override one step. For that case each Node body's setup also exists as a composite action, which runs **in your own job**: @@ -395,6 +435,9 @@ commands do not need an authenticated `git fetch` afterwards. Branch pushes compare against the commit before the push, with history fetched by checkout. `force: true` returns every filter name without a checkout. Keep the filters in the calling repository, including shared contracts and generators. +With `fingerprint: true` the diff is not consulted at all: a blob-less, depth-1 +fetch of the tree under test feeds the fingerprints, and the job additionally +needs `statuses: read`. `_ci-rust.yml` finds `rust-toolchain.toml` or `rust-toolchain` from its `working-directory` up to the repository root and installs that toolchain. Without diff --git a/scripts/workflow-contracts.test.ts b/scripts/workflow-contracts.test.ts index e80b7cc..43f8542 100644 --- a/scripts/workflow-contracts.test.ts +++ b/scripts/workflow-contracts.test.ts @@ -11,9 +11,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; -const workflow = readFileSync('.github/workflows/_ci-rust.yml', 'utf8'); -function script(name: string): string { - const step = workflow +function script(name: string, file = '_ci-rust.yml'): string { + const step = readFileSync(`.github/workflows/${file}`, 'utf8') .split(` - name: ${name}\n`)[1] ?.split('\n - ')[0]; const body = step?.split(' run: |\n')[1]; @@ -356,3 +355,268 @@ test('optional aggregate gates remain on by default and reject failed or cancell } } }); + +const AREA_FILTERS = [ + 'tools:', + ' - tools/**', + " - '*'", + 'core:', + ' - services/core/**/*.json # comment', + ' - "packages/**/Capacity.java"', + 'negated:', + ' - tools/**', + " - '!tools/docs/**'", + 'braced:', + " - 'src/**/*.{ts,js}'", + '' +].join('\n'); + +function git(cwd: string, ...args: string[]): string { + const run = spawnSync('git', args, { + cwd, + encoding: 'utf8', + env: { + ...process.env, + GIT_AUTHOR_NAME: 't', + GIT_AUTHOR_EMAIL: 't@t', + GIT_COMMITTER_NAME: 't', + GIT_COMMITTER_EMAIL: 't@t', + GIT_CONFIG_GLOBAL: '/dev/null' + } + }); + assert.equal(run.status, 0, run.stderr); + return run.stdout.trim(); +} + +function fingerprints(root: string, sha: string): Record { + const output = join(root, 'output'); + writeFileSync(output, ''); + const run = spawnSync( + 'bash', + ['-c', script('Compute the area fingerprints', '_detect-changes.yml')], + { + env: { + ...process.env, + RUNNER_TEMP: root, + GITHUB_OUTPUT: output, + FILTERS: AREA_FILTERS, + TOKEN: '', + SERVER_URL: `file://${root}`, + REPOSITORY: 'source', + SHA: sha + }, + encoding: 'utf8' + } + ); + assert.equal(run.status, 0, run.stderr); + const lines = readFileSync(output, 'utf8').trim().split('\n'); + assert.equal( + lines[0], + 'names=["tools","core","negated","braced"]', + 'every filter is named, fingerprinted or not' + ); + const line = lines.find((entry) => entry.startsWith('fingerprints=')); + assert.ok(line, run.stdout); + return JSON.parse(line.slice('fingerprints='.length)); +} + +test('area fingerprints follow the matched content and skip filters git cannot reproduce', () => { + const root = mkdtempSync(join(tmpdir(), 'workflow-fingerprint-')); + const source = join(root, 'source'); + const write = (path: string, content: string) => { + mkdirSync(join(source, path, '..'), { recursive: true }); + writeFileSync(join(source, path), content); + }; + try { + mkdirSync(source); + git(source, 'init', '-q'); + git(source, 'config', 'uploadpack.allowFilter', 'true'); + git(source, 'config', 'uploadpack.allowAnySHA1InWant', 'true'); + write('package.json', '{}'); + write('tools/cli/run.ts', 'one'); + write('services/core/app/config.json', '{}'); + write('services/core/app/Model.php', ' { + git(source, 'add', '-A'); + git( + source, + '-c', + 'user.name=t', + '-c', + 'user.email=t@t', + 'commit', + '-qm', + message + ); + return git(source, 'rev-parse', 'HEAD'); + }; + + const first = fingerprints(root, commit('first')); + assert.deepEqual(Object.keys(first), ['tools', 'core']); + for (const value of Object.values(first)) + assert.match(value, /^[0-9a-f]{64}$/); + + write('services/core/app/Model.php', ' { + const root = mkdtempSync(join(tmpdir(), 'workflow-fingerprint-open-')); + try { + assert.deepEqual(fingerprints(root, '0'.repeat(40)), {}); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('an area runs unless forced off its exact green fingerprint', () => { + const root = mkdtempSync(join(tmpdir(), 'workflow-fingerprint-decide-')); + const output = join(root, 'output'); + const fp = (char: string) => char.repeat(64); + const decide = (env: Record) => { + writeFileSync(output, ''); + const run = spawnSync( + 'bash', + ['-c', script('Decide which areas run', '_detect-changes.yml')], + { + env: { + ...process.env, + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: join(root, 'summary'), + NAMES: '["tools","core","bots","raw"]', + FINGERPRINTS: JSON.stringify({ + tools: fp('a'), + core: fp('b'), + bots: fp('c') + }), + GREEN: JSON.stringify([ + { context: 'ci-area/tools', description: fp('a') }, + { context: 'ci-area/core', description: fp('f') }, + { context: 'ci-area/elsewhere', description: fp('c') } + ]), + FORCE: 'false', + ...env + }, + encoding: 'utf8' + } + ); + assert.equal(run.status, 0, run.stderr); + return readFileSync(output, 'utf8').trim(); + }; + try { + assert.equal( + decide({}), + 'changes=["core","bots","raw"]\nverified=["tools"]', + 'only the exact context and fingerprint count' + ); + assert.equal( + decide({ FORCE: 'true' }), + 'changes=["tools","core","bots","raw"]\nverified=[]' + ); + assert.equal( + decide({ GREEN: '', FINGERPRINTS: '' }), + 'changes=["tools","core","bots","raw"]\nverified=[]', + 'a failed lookup runs everything' + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('recording writes one status per named area and refuses malformed input', () => { + const root = mkdtempSync(join(tmpdir(), 'workflow-fingerprint-record-')); + const calls = join(root, 'calls'); + writeFileSync( + join(root, 'curl'), + ['#!/bin/sh', 'echo "$* $(cat)" >> "$CALLS"', ''].join('\n'), + { mode: 0o755 } + ); + const fp = 'd'.repeat(64); + const record = (env: Record) => { + writeFileSync(calls, ''); + const run = spawnSync( + 'bash', + [ + '-c', + script( + 'Mark the green areas on this commit', + '_record-area-fingerprints.yml' + ) + ], + { + env: { + ...process.env, + PATH: `${root}:${process.env.PATH}`, + CALLS: calls, + TOKEN: 't', + API_URL: 'https://api.example', + REPOSITORY: 'o/r', + HEAD_REPOSITORY: 'o/r', + SHA: 'abc', + RUN_URL: 'https://run', + FINGERPRINTS: JSON.stringify({ tools: fp, core: fp }), + AREAS: '*', + ...env + }, + encoding: 'utf8' + } + ); + return { + status: run.status, + calls: readFileSync(calls, 'utf8').trim().split('\n').filter(Boolean) + }; + }; + try { + const all = record({}); + assert.equal(all.status, 0); + assert.equal(all.calls.length, 2); + assert.match(all.calls[0], /\/repos\/o\/r\/statuses\/abc /); + assert.match(all.calls[0], /"context":"ci-area\/tools"/); + assert.match(all.calls[0], new RegExp(`"description":"${fp}"`)); + + const named = record({ AREAS: '["core","unknown"]' }); + assert.equal(named.status, 0); + assert.equal(named.calls.length, 1); + assert.match(named.calls[0], /ci-area\/core/); + + assert.deepEqual(record({ HEAD_REPOSITORY: 'fork/r' }), { + status: 0, + calls: [] + }); + assert.deepEqual(record({ FINGERPRINTS: '' }), { status: 0, calls: [] }); + assert.equal(record({ AREAS: 'tools' }).status, 1); + assert.equal( + record({ FINGERPRINTS: JSON.stringify({ tools: 'nope' }) }).status, + 1 + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +});