Repository navigation
Verify cache seeds without gating deploy smoke jobs #435
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| #!/usr/bin/env bash | ||
| # Mutate the workflow itself so contract checks cannot merely test a detached if. | ||
| set -euo pipefail | ||
| ACTION_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) | ||
| WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/edgezero-cache-publication-test.XXXXXX") | ||
| trap 'rm -rf -- "$WORK_DIR"' EXIT | ||
| # shellcheck source=common.sh | ||
| source "$ACTION_DIR/tests/common.sh" | ||
| workflow="$ACTION_DIR/../../workflows/deploy-action.yml" | ||
|
|
||
| reject_mutation() { | ||
| local name="$1" expression="$2" diagnostic="$3" status=0 | ||
| yq "$expression" "$workflow" >"$WORK_DIR/workflow.yml" | ||
| CACHE_WORKFLOW_FILE="$WORK_DIR/workflow.yml" bash "$ACTION_DIR/tests/fixture.sh" >"$WORK_DIR/log" 2>&1 || status=$? | ||
| [[ "$status" == 1 ]] || fail "workflow guard accepted mutation: $name (exit $status)" | ||
| grep -Fq "$diagnostic" "$WORK_DIR/log" || { | ||
| cat "$WORK_DIR/log" >&2 | ||
| fail "mutation failed for an unrelated reason: $name" | ||
| } | ||
| } | ||
|
|
||
| for kind in source target; do | ||
| publication=".jobs.\"cache-restore-seed\".steps[] | select(.name == \"Require exact $kind seed publication\")" | ||
| reject_mutation 'missing final gate' \ | ||
| ".jobs.\"cache-restore-seed\".steps |= map(select(.name != \"Require exact $kind seed publication\"))" \ | ||
| 'missing exact publication assertion' | ||
| reject_mutation 'hardcoded first hit' \ | ||
| "($publication | .env.CACHE_SEED_FIRST_HIT) = \"true\"" \ | ||
| 'publication assertion ignores first lookup output' | ||
| reject_mutation 'hardcoded second hit' \ | ||
| "($publication | .env.CACHE_SEED_SECOND_HIT) = \"true\"" \ | ||
| 'publication assertion ignores second lookup output' | ||
| reject_mutation 'skipped final gate' \ | ||
| "($publication | .if) = \"false\"" \ | ||
| 'publication assertion can be skipped' | ||
| reject_mutation 'ignored final failure' \ | ||
| "($publication | .\"continue-on-error\") = true" \ | ||
| 'publication assertion failure is ignored' | ||
| reject_mutation 'gate always succeeds' \ | ||
| "($publication | .run) = \"true\"" \ | ||
| 'publication assertion accepts unverified outputs' | ||
| reject_mutation 'gate always fails' \ | ||
| "($publication | .run) = \"false\"" \ | ||
| 'publication assertion rejects a verified exact hit' | ||
| for step in "$kind-save-1" "$kind-verify-1" "$kind-save-2" "$kind-verify-2"; do | ||
| for input in path key; do | ||
| reject_mutation "$step wrong $input" \ | ||
| "(.jobs.\"cache-restore-seed\".steps[] | select(.id == \"$step\") | .with.$input) = \"wrong\"" \ | ||
| "$step uses a different cache $input" | ||
| done | ||
| if [[ "$step" == *-2 ]]; then | ||
| reject_mutation "$step wrong retry condition" \ | ||
| "(.jobs.\"cache-restore-seed\".steps[] | select(.id == \"$step\") | .if) = \"false\"" \ | ||
| "$step retries without a first-attempt miss" | ||
| else | ||
| reject_mutation "$step skipped initial attempt" \ | ||
| "(.jobs.\"cache-restore-seed\".steps[] | select(.id == \"$step\") | .if) = \"false\"" \ | ||
| "$step skips the initial publication attempt" | ||
| fi | ||
| done | ||
| done | ||
| printf 'source and target publication workflow mutation contracts passed\n' | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #!/usr/bin/env bash | ||
| set -euo pipefail | ||
|
|
||
| # Publish each target before downstream jobs test first-build compiler reuse. | ||
| fixture="${GITHUB_WORKSPACE:?}/cache-restore-fixture" | ||
| target_dir=$(realpath "${CARGO_TARGET_DIR:?}") | ||
|
Collaborator
There was a problem hiding this comment. 😃 Directory cleanup is safely confined: the target path is resolved and must be exactly the fixture directory, unsafe host target names are rejected, and symlinked target or |
||
| expected="$(realpath "${RUNNER_TEMP:?}")/edgezero-rust-cache/cache-restore-smoke-v2/target" | ||
| [[ "$target_dir" == "$expected" ]] || { | ||
| printf '::error::compiler seed must use its confined fixture target directory\n' >&2 | ||
| exit 1 | ||
| } | ||
| cd "$fixture" | ||
| host=$(rustc -vV | sed -n 's/^host: //p') | ||
| [[ "$host" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ ]] || { | ||
| printf '::error::rustc must report a safe host target component\n' >&2 | ||
| exit 1 | ||
| } | ||
| for target in "$host" wasm32-wasip1; do | ||
| for attempt in 1 2 3; do | ||
| # Cargo must call sccache again after a failed publication, rather than | ||
| # reporting locally fresh outputs as a successful cache seed. | ||
| debug_dir="$target_dir/$target/debug" | ||
| [[ ! -L "$target_dir/$target" && ! -L "$debug_dir" ]] || { | ||
| printf '::error::compiler seed cleanup must not follow target symlinks\n' >&2 | ||
| exit 1 | ||
| } | ||
| rm -rf -- "$debug_dir" | ||
| sccache --zero-stats | ||
| cargo build --locked --lib --target "$target" | ||
| stats=$(sccache --show-stats --stats-format json) | ||
| printf 'Compiler seed for %s (attempt %s): %s\n' "$target" "$attempt" "$stats" | ||
| if jq -e '((.stats.cache_hits.counts.Rust // 0) + (.stats.cache_writes // 0)) > 0 and (.stats.cache_errors.counts.Rust // 0) == 0 and (.stats.cache_read_errors // 0) == 0 and (.stats.cache_write_errors // 0) == 0' <<<"$stats" >/dev/null; then | ||
| break | ||
| fi | ||
|
Comment on lines
+32
to
+34
Collaborator
There was a problem hiding this comment. [P2] Verify compiler publication with a remote read-back hit A successful write counter does not establish publication with the pinned backend. sccache v0.16.0 uses OpenDAL 0.55.0, whose GHAC-v2 writer discards errors from FinalizeCacheEntryUpload. When uploads succeed but finalization returns 503, cache_writes becomes 1 while cache_write_errors remains 0. Reproduced with the real pinned binary and Rust 1.98.1 against a local cache-v2 service: both targets were accepted on attempt 1 and the seed exited 0, but the fresh-target compiler-reuse check exited 1. This bypasses the new retries for unpublished entries and leaves the downstream smoke job failing. Require a bounded remote read-back hit before accepting a new write, and add finalization-failure coverage. |
||
| if [[ "$attempt" == 3 ]]; then | ||
| printf '::error::failed to publish compiler cache for %s after three attempts\n' "$target" >&2 | ||
| exit 1 | ||
| fi | ||
| printf '::warning::compiler cache seed failed; retrying after backoff\n' | ||
| sleep "$((attempt * 60))" | ||
| done | ||
| done | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| #!/usr/bin/env bash | ||
| set -euo pipefail | ||
| ACTION_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) | ||
| WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/edgezero-cache-seed-test.XXXXXX") | ||
| trap 'rm -rf -- "$WORK_DIR"' EXIT | ||
| # shellcheck source=common.sh | ||
| source "$ACTION_DIR/tests/common.sh" | ||
| mkdir -p "$WORK_DIR/bin" "$WORK_DIR/workspace/cache-restore-fixture" "$WORK_DIR/runner" | ||
| export GITHUB_WORKSPACE="$WORK_DIR/workspace" RUNNER_TEMP="$WORK_DIR/runner" | ||
| export CARGO_TARGET_DIR="$RUNNER_TEMP/edgezero-rust-cache/cache-restore-smoke-v2/target" | ||
| export PROBE_LOG="$WORK_DIR/calls" PROBE_COUNT="$WORK_DIR/count" | ||
| cat >"$WORK_DIR/bin/rustc" <<'MOCK' | ||
| #!/usr/bin/env bash | ||
| printf 'host: %s\n' "${PROBE_HOST:-test-host}" | ||
| MOCK | ||
| cat >"$WORK_DIR/bin/sleep" <<'MOCK' | ||
| #!/usr/bin/env bash | ||
| printf 'sleep %s\n' "$*" >>"$PROBE_LOG" | ||
| MOCK | ||
| cat >"$WORK_DIR/bin/cargo" <<'MOCK' | ||
| #!/usr/bin/env bash | ||
| printf 'build %s\n' "$*" >>"$PROBE_LOG" | ||
| [[ "${PROBE_MODE:-}" != compile-failure ]] || exit 42 | ||
| [[ "$PROBE_MODE" != compile-wasm-failure || "${*: -1}" != wasm32-wasip1 ]] || exit 42 | ||
| [[ ! -d "$CARGO_TARGET_DIR/${*: -1}/debug" ]] || exit 43 | ||
| mkdir -p "$CARGO_TARGET_DIR/${*: -1}/debug" | ||
| MOCK | ||
| cat >"$WORK_DIR/bin/sccache" <<'MOCK' | ||
| #!/usr/bin/env bash | ||
| if [[ "$1" == --zero-stats ]]; then | ||
| printf 'zero\n' >>"$PROBE_LOG" | ||
| exit 0 | ||
| fi | ||
| [[ "$1" == --show-stats ]] || exit 2 | ||
| count=$(cat "$PROBE_COUNT") | ||
| printf '%s\n' "$((count + 1))" >"$PROBE_COUNT" | ||
| if [[ "$PROBE_MODE" == permanent || | ||
| ( "$PROBE_MODE" == transient && "$count" == 0 ) || | ||
| ( "$PROBE_MODE" == wasm-transient && "$count" == 1 ) || | ||
| ( "$PROBE_MODE" == both-transient && ( "$count" == 0 || "$count" == 2 ) ) || | ||
| ( "$PROBE_MODE" == wasm-permanent && "$count" -ge 1 ) ]]; then | ||
| printf '{"stats":{"cache_write_errors":1,"cache_writes":0}}\n' | ||
| elif [[ "$PROBE_MODE" == read-error ]]; then | ||
| printf '{"stats":{"cache_writes":1,"cache_read_errors":1}}\n' | ||
| elif [[ "$PROBE_MODE" == compile-cache-error ]]; then | ||
| printf '{"stats":{"cache_hits":{"counts":{"Rust":1}},"cache_errors":{"counts":{"Rust":1}}}}\n' | ||
| elif [[ "$PROBE_MODE" == malformed ]]; then | ||
| printf 'not-json\n' | ||
| elif [[ "$PROBE_MODE" == empty ]]; then | ||
| printf '{"stats":{}}\n' | ||
| elif [[ "$PROBE_MODE" == hit ]]; then | ||
| printf '{"stats":{"cache_hits":{"counts":{"Rust":1}}}}\n' | ||
| else | ||
| printf '{"stats":{"cache_writes":1}}\n' | ||
| fi | ||
| MOCK | ||
| chmod +x "$WORK_DIR/bin/"* | ||
| export PATH="$WORK_DIR/bin:$PATH" | ||
| run_seed() { | ||
| rm -rf "$CARGO_TARGET_DIR" | ||
| mkdir -p "$CARGO_TARGET_DIR" | ||
| printf '0\n' >"$PROBE_COUNT" | ||
| : >"$PROBE_LOG" | ||
| PROBE_MODE="$1" bash "$ACTION_DIR/tests/seed-compiler-cache.sh" >"$WORK_DIR/log" 2>&1 | ||
| } | ||
| run_seed transient || fail 'compiler seed does not recover from a transient write failure' | ||
| [[ "$(grep -c '^build ' "$PROBE_LOG")" == 3 ]] || fail 'retry does not rebuild fresh outputs' | ||
| [[ "$(grep -c '^sleep ' "$PROBE_LOG")" == 1 ]] || fail 'transient error does not back off' | ||
| run_seed hit || fail 'existing compiler hit is not accepted as published' | ||
| [[ "$(grep -c '^build ' "$PROBE_LOG")" == 2 ]] || fail 'successful seed does not build both targets' | ||
| grep -q -- '--target test-host$' "$PROBE_LOG" || fail 'successful seed omits native target' | ||
| grep -q -- '--target wasm32-wasip1$' "$PROBE_LOG" || fail 'successful seed omits WASM target' | ||
| [[ "$(grep -c '^zero$' "$PROBE_LOG")" == 2 ]] || fail 'successful seed does not reset stats for each target' | ||
| for mode in wasm-transient both-transient; do | ||
| run_seed "$mode" || fail "$mode does not recover" | ||
| expected=3 | ||
| if [[ "$mode" == both-transient ]]; then expected=4; fi | ||
| [[ "$(grep -c '^build ' "$PROBE_LOG")" == "$expected" ]] || fail "$mode does not rebuild every attempt" | ||
| [[ "$(grep -c '^zero$' "$PROBE_LOG")" == "$expected" ]] || fail "$mode does not reset every attempt" | ||
| done | ||
| for mode in permanent empty compile-failure compile-wasm-failure wasm-permanent read-error compile-cache-error malformed; do | ||
| if run_seed "$mode"; then fail "accepted unsuccessful compiler seeding: $mode"; fi | ||
| case "$mode" in | ||
| compile-failure) expected=1 ;; | ||
| compile-wasm-failure) expected=2 ;; | ||
| wasm-permanent) expected=4 ;; | ||
| *) expected=3 ;; | ||
| esac | ||
| [[ "$(grep -c '^build ' "$PROBE_LOG")" == "$expected" ]] || fail "$mode violates per-target retry limits" | ||
| [[ "$(grep -c '^zero$' "$PROBE_LOG")" == "$expected" ]] || fail "$mode uses stale stats" | ||
| done | ||
| # Cleanup must never follow restored symlinks outside the fixture root. | ||
| for target in test-host wasm32-wasip1; do | ||
| for component in namespace debug; do | ||
| rm -rf -- "$CARGO_TARGET_DIR" "$WORK_DIR/outside" | ||
| mkdir -p "$CARGO_TARGET_DIR" "$WORK_DIR/outside/debug" | ||
| printf keep >"$WORK_DIR/outside/debug/marker" | ||
| if [[ "$component" == namespace ]]; then | ||
| ln -s "$WORK_DIR/outside" "$CARGO_TARGET_DIR/$target" | ||
| else | ||
| mkdir -p "$CARGO_TARGET_DIR/$target" | ||
| ln -s "$WORK_DIR/outside/debug" "$CARGO_TARGET_DIR/$target/debug" | ||
| fi | ||
| status=0 | ||
| PROBE_MODE=hit bash "$ACTION_DIR/tests/seed-compiler-cache.sh" >"$WORK_DIR/log" 2>&1 || status=$? | ||
| [[ -f "$WORK_DIR/outside/debug/marker" ]] || fail "cleanup escaped through $target $component symlink" | ||
| [[ "$status" != 0 ]] || fail "accepted $target $component symlink" | ||
| grep -Fq 'compiler seed cleanup must not follow target symlinks' "$WORK_DIR/log" || fail 'symlink case failed for an unrelated reason' | ||
| done | ||
| done | ||
| rm -rf -- "$CARGO_TARGET_DIR" | ||
| mkdir -p "$CARGO_TARGET_DIR" "$WORK_DIR/runner/edgezero-rust-cache/debug" | ||
| printf keep >"$WORK_DIR/runner/edgezero-rust-cache/debug/marker" | ||
| if PROBE_HOST=../.. PROBE_MODE=hit bash "$ACTION_DIR/tests/seed-compiler-cache.sh" >"$WORK_DIR/log" 2>&1; then | ||
| fail 'accepted traversal in compiler host' | ||
| fi | ||
| [[ -f "$WORK_DIR/runner/edgezero-rust-cache/debug/marker" ]] || fail 'removed output through host traversal' | ||
| grep -Fq 'rustc must report a safe host target component' "$WORK_DIR/log" || fail 'traversal case failed for an unrelated reason' | ||
| # A mistaken directory must never be removed by retry cleanup. | ||
| export CARGO_TARGET_DIR="$WORK_DIR/outside" | ||
| mkdir -p "$CARGO_TARGET_DIR" | ||
| printf keep >"$CARGO_TARGET_DIR/marker" | ||
| if PROBE_MODE=hit bash "$ACTION_DIR/tests/seed-compiler-cache.sh" >"$WORK_DIR/log" 2>&1; then | ||
| fail 'accepted unconfined compiler seed directory' | ||
| fi | ||
| [[ -f "$CARGO_TARGET_DIR/marker" ]] || fail 'removed unconfined output' | ||
| printf 'compiler seed backoff contracts passed\n' |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
😃 Workflow guards are themselves tested by breaking the workflow: making 38 deliberate edits to the workflow and requiring each one to fail
fixture.shwith the matching error message proves the guards actually catch the problems, not just that they run.