From 9f4c76e45d0491562bf7bd022596e5af6c82d6c3 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Tue, 27 Oct 2026 11:56:23 +0000 Subject: [PATCH 1/3] test(mutationlab): run the lab on a cluster of its own The lab swapped its image into the product's controller Deployment on the e2e cluster and borrowed its admission registration, serving certificate, and Service. fd6f6933 removed the admission webhook, and the lab lost its admission source and the conversion webhook's CA while every check stayed green, because nothing ran the lab. The lab now runs on its own k3d cluster, gitops-reverser-capture-lab, with nothing of the product on it. Every source arrives through its real mechanism: the API server's audit webhook backend posts to a NodePort over mutual TLS (the lab now requires the client certificate), a ValidatingWebhookConfiguration of its own sends admission reviews, the Widget CRD's conversion webhook calls the lab's Service, the wardle sample API server serves flunders, and the lab watches with read-only RBAC. lab-certs issues the CA, serving certificate, and the API server's client certificate before the cluster exists, so the audit kubeconfig is final at startup and the node never restarts. The registration keeps the match rules and the name the corpus was captured under: the API server keys the recorder's audit annotation by the webhook's name, so renaming it changes 13 corpus rows. test/mutationlab/Taskfile-lab.yml carries lab-bootstrap, lab-capture, lab-corpus-update, lab-status, and lab-cleanup. The cluster can run beside e2e: its own stamps, lock, and port, and start-cluster.sh gains K3D_SWITCH_CONTEXT=false so creating it never moves the current context. lab-bootstrap ends with TestSourcesLive, which proves each of the five sources with a probe write; TestClusterProvenance fails a capture on another server version than CLUSTER.md records, and writes those rows on an update. selfcontained_test.go keeps the lab off product tasks, product resource names, and unshared paths, and mutation-lab.yml runs lab-capture nightly. lab-e2e, _lab-deployed, swap-image.sh, and hack/e2e/lab-port-forward.sh are removed. The sample API server gains a kustomization.yaml that both clusters apply; the e2e render is byte-identical. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mutation-lab.yml | 78 +++++ Taskfile.yml | 4 + cmd/mutation-capture-lab/main.go | 14 +- docs/ci-overview.md | 3 +- .../mutation-capture-lab-own-cluster.md | 38 ++- docs/proud.md | 3 +- docs/spec/mutation-capture-lab-design.md | 42 +-- docs/tasks-overview.md | 6 +- hack/e2e/lab-port-forward.sh | 84 ------ internal/mutationlab/labserver/api.go | 3 +- test/e2e/Taskfile-e2e.yml | 71 ----- test/e2e/cluster/README.md | 4 + test/e2e/cluster/start-cluster.sh | 18 +- test/e2e/setup/manifests/kustomization.yaml | 10 +- .../sample-apiserver/kustomization.yaml | 16 ++ test/mutationlab/Dockerfile | 4 +- test/mutationlab/README.md | 83 +++--- test/mutationlab/Taskfile-lab.yml | 211 ++++++++++++++ test/mutationlab/deploy.sh | 113 ++++++++ test/mutationlab/deploy/deployment.yaml | 75 +++++ test/mutationlab/deploy/kustomization.yaml | 13 + test/mutationlab/deploy/namespace.yaml | 4 + test/mutationlab/deploy/rbac.yaml | 38 +++ test/mutationlab/deploy/service.yaml | 35 +++ .../deploy/validating-webhook.yaml | 36 +++ .../e2e/cluster_provenance_test.go | 110 +++++++ test/mutationlab/e2e/crd_conversion_test.go | 79 +++--- test/mutationlab/e2e/doc.go | 3 +- test/mutationlab/e2e/harness_test.go | 22 +- test/mutationlab/e2e/sources_live_test.go | 122 ++++++++ .../e2e/workload_scenarios_test.go | 7 +- test/mutationlab/port-forward.sh | 56 ++++ test/mutationlab/selfcontained_test.go | 175 ++++++++++++ test/mutationlab/swap-image.sh | 76 ----- test/mutationlab/tools/lab-certs/main.go | 268 ++++++++++++++++++ test/mutationlab/tools/lab-certs/main_test.go | 167 +++++++++++ 36 files changed, 1718 insertions(+), 373 deletions(-) create mode 100644 .github/workflows/mutation-lab.yml delete mode 100755 hack/e2e/lab-port-forward.sh create mode 100644 test/e2e/setup/manifests/sample-apiserver/kustomization.yaml create mode 100644 test/mutationlab/Taskfile-lab.yml create mode 100644 test/mutationlab/deploy.sh create mode 100644 test/mutationlab/deploy/deployment.yaml create mode 100644 test/mutationlab/deploy/kustomization.yaml create mode 100644 test/mutationlab/deploy/namespace.yaml create mode 100644 test/mutationlab/deploy/rbac.yaml create mode 100644 test/mutationlab/deploy/service.yaml create mode 100644 test/mutationlab/deploy/validating-webhook.yaml create mode 100644 test/mutationlab/e2e/cluster_provenance_test.go create mode 100644 test/mutationlab/e2e/sources_live_test.go create mode 100644 test/mutationlab/port-forward.sh create mode 100644 test/mutationlab/selfcontained_test.go delete mode 100755 test/mutationlab/swap-image.sh create mode 100644 test/mutationlab/tools/lab-certs/main.go create mode 100644 test/mutationlab/tools/lab-certs/main_test.go diff --git a/.github/workflows/mutation-lab.yml b/.github/workflows/mutation-lab.yml new file mode 100644 index 000000000..b24284bf4 --- /dev/null +++ b/.github/workflows/mutation-lab.yml @@ -0,0 +1,78 @@ +# Mutation-capture lab: creates the lab's own k3d cluster, runs every scenario, and fails on any +# difference from the committed corpus under test/mutationlab/corpus/ +# (docs/design/mutation-capture-lab-own-cluster.md). +# +# Nightly and on demand rather than per PR. Its job is to notice when something stops the lab +# from working, or makes it record something else, before the next Kubernetes upgrade depends on +# it; a break shows up within a day of the change that caused it. Nothing here publishes, and the +# run needs no secrets beyond a read token to pull the CI container. +name: Mutation lab + +on: + schedule: + - cron: "17 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: mutation-lab-${{ github.ref }} + cancel-in-progress: true + +env: + # The CI base container main last published (ci.yml moves :latest only from main): the same + # Go, kubectl, k3d, and task the e2e legs run with. + CI_CONTAINER: ghcr.io/configbutler/gitops-reverser-ci:latest + CI_WORKDIR: /workspaces/${{ github.event.repository.name }} + +jobs: + capture: + name: Capture and compare to the corpus + runs-on: ubuntu-latest + timeout-minutes: 40 + permissions: + contents: read + packages: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Login to Docker registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Run lab-capture in the CI container + # One container for the whole run: the lab's kube context and its records-API forward + # live only as long as it does. Host networking reaches the k3d API port and the forward. + run: | + docker run --rm \ + --network host \ + -v "${GITHUB_WORKSPACE}:${{ env.CI_WORKDIR }}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -w "${{ env.CI_WORKDIR }}" \ + -e HOST_PROJECT_PATH=${{ github.workspace }} \ + ${{ env.CI_CONTAINER }} \ + bash -c " + git config --global --add safe.directory ${{ env.CI_WORKDIR }} + task lab-capture + " + + - name: Show the lab on failure + if: failure() + run: | + docker run --rm \ + --network host \ + -v "${GITHUB_WORKSPACE}:${{ env.CI_WORKDIR }}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -w "${{ env.CI_WORKDIR }}" \ + ${{ env.CI_CONTAINER }} \ + bash -c " + k3d kubeconfig merge gitops-reverser-capture-lab --kubeconfig-switch-context=false \ + --kubeconfig-merge-default >/dev/null 2>&1 || exit 0 + kubectl --context k3d-gitops-reverser-capture-lab get pods -A -o wide || true + kubectl --context k3d-gitops-reverser-capture-lab -n mutation-capture-lab \ + logs deployment/mutation-capture-lab --tail=200 || true + " diff --git a/Taskfile.yml b/Taskfile.yml index 02ac12fc1..f8c8c7c2f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -24,6 +24,10 @@ includes: taskfile: ./test/e2e/Taskfile-e2e.yml dir: . flatten: true + lab: + taskfile: ./test/mutationlab/Taskfile-lab.yml + dir: . + flatten: true tasks: default: diff --git a/cmd/mutation-capture-lab/main.go b/cmd/mutation-capture-lab/main.go index cf18fcbb0..cb0d94249 100644 --- a/cmd/mutation-capture-lab/main.go +++ b/cmd/mutation-capture-lab/main.go @@ -6,9 +6,11 @@ // deliberately NOT a second GitOps Reverser; see // docs/spec/mutation-capture-lab-design.md. // -// It deliberately serves the SAME webhook URLs as the product -// (/validate-all, /audit-webhook) so a lab deployment can swap the product image -// without reconfiguring the cluster's admission/audit wiring. +// It runs on a cluster of its own (test/mutationlab/Taskfile-lab.yml), which +// delivers to it the way any cluster delivers: the API server's audit webhook +// backend posts to /audit-webhook over mutual TLS, a ValidatingWebhookConfiguration +// sends admission reviews to /validate-all, and the scenario CRD's conversion +// webhook calls /convert. package main import ( @@ -121,12 +123,6 @@ func buildServers( auditMux := http.NewServeMux() auditMux.Handle("/audit-webhook", audit) - // Audit routes are NAMED in the e2e cluster: its bootstrap kubeconfig posts to - // /audit-webhook/ (the chart's local KubeProvider), because the bare - // path is the product's shared annotation-routed endpoint. The lab records what - // arrives regardless of which source it was addressed to, so it takes the whole - // subtree — an exact-path-only mux 404s every event the cluster sends. - auditMux.Handle("/audit-webhook/", audit) return []server{ {name: "admission", certDir: cfg.admissionCert, srv: &http.Server{ diff --git a/docs/ci-overview.md b/docs/ci-overview.md index e25464e26..3f5547e4c 100644 --- a/docs/ci-overview.md +++ b/docs/ci-overview.md @@ -20,6 +20,7 @@ so local and CI results cannot drift apart. | Release & publish | [release.yml](../.github/workflows/release.yml) tail jobs | after trusted validation is green | `GITHUB_TOKEN` + OIDC | packages, releases, attestations | Version (release-please), **retag** the CI-built multi-arch digests to semver + `latest` (zero rebuilds), publish chart, sign, attest | | Hygiene | [scorecard.yml](../.github/workflows/scorecard.yml) | weekly + `main` | none | security-events | OpenSSF Scorecard supply-chain checks | | Hygiene | [pr-title.yml](../.github/workflows/pr-title.yml) | `pull_request` (including `edited`) | none | none | The PR title is a Conventional Commit, since it becomes the squash-merge subject release-please classifies | +| Hygiene | [mutation-lab.yml](../.github/workflows/mutation-lab.yml) | nightly + manual | `GITHUB_TOKEN` (read) | none | The mutation-capture lab on its own cluster, compared to its committed corpus ([design](design/mutation-capture-lab-own-cluster.md)) | Three properties are worth calling out: @@ -164,5 +165,5 @@ GitHub's OIDC identity and logged in the public Rekor transparency log. | Tool versions | [.devcontainer/Dockerfile](../.devcontainer/Dockerfile) (single source for devcontainer *and* CI) | | Validation pipeline | [.github/workflows/ci.yml](../.github/workflows/ci.yml) | | Release pipeline | [.github/workflows/release.yml](../.github/workflows/release.yml), [.github/workflows/release-pr.yml](../.github/workflows/release-pr.yml) | -| Hygiene | [.github/workflows/scorecard.yml](../.github/workflows/scorecard.yml), [.github/dependabot.yml](../.github/dependabot.yml) | +| Hygiene | [.github/workflows/scorecard.yml](../.github/workflows/scorecard.yml), [.github/workflows/mutation-lab.yml](../.github/workflows/mutation-lab.yml), [.github/dependabot.yml](../.github/dependabot.yml) | | Release process details | [.github/RELEASES.md](../.github/RELEASES.md) | diff --git a/docs/design/mutation-capture-lab-own-cluster.md b/docs/design/mutation-capture-lab-own-cluster.md index bcbeb4f0e..83776658b 100644 --- a/docs/design/mutation-capture-lab-own-cluster.md +++ b/docs/design/mutation-capture-lab-own-cluster.md @@ -1,6 +1,6 @@ # Mutation-capture lab on its own cluster -Status: proposed, 2026-10-27. Not built. +Status: built, 2026-10-27. [As built](#as-built) lists where the build departs from this design. The [mutation-capture lab](../spec/mutation-capture-lab-design.md) records what Kubernetes emits through watches, audit, admission, CRD conversion, and an aggregated API, and commits it as a @@ -15,7 +15,7 @@ the requirements that keep the capture faithful and the lab working. ## What broke, and why it could The lab never owned its wiring. `task lab-e2e` swapped the lab image into the product's controller -Deployment on the e2e cluster ([`swap-image.sh`](../../test/mutationlab/swap-image.sh)) and borrowed +Deployment on the e2e cluster (`swap-image.sh`, since removed) and borrowed everything around it: | The lab needs | It came from | Since `fd6f6933` | @@ -181,7 +181,7 @@ context rather than the current one. Otherwise bootstrapping the lab would move | the lab binary, recorders, normalizer, corpus, and driver | unchanged, apart from the client-CA flag, the context, and the readiness check | | the content-hashed lab image and its build | unchanged | | `lab-port-forward.sh` | with the lab's namespace and selector | -| the audit kubeconfig template | from `generate-audit-webhook-kubeconfig.sh`, fed from files instead of Secrets | +| the audit kubeconfig | written by the certificate command, not by `generate-audit-webhook-kubeconfig.sh` (see [As built](#as-built)) | Not reused: `inject-webhook-tls.sh`, whose node restart the lab avoids, and the Flux stack. @@ -230,10 +230,28 @@ Each one names the failure it prevents. 7. The scheduled CI job, the static self-containment check, and the lab's README and [spec](../spec/mutation-capture-lab-design.md#isolated-test-setup) rewritten to match. -## Open questions - -- **CI cadence.** Nightly plus manual dispatch is the proposal. A per-PR job would catch a break - in the PR that causes it, for a few runner minutes per PR. -- **Memory beside e2e.** One k3s node with the lab and the sample API server should stay near - 1 GiB, but that is an estimate until measured. If it is more, `lab-capture` can take the e2e lock - as well, so the two run in turn. +## As built + +- **Two Services, not one.** `mutation-capture-lab` is a ClusterIP Service for admission, + conversion (443 to 9443), and the records API (8080); `mutation-capture-lab-audit` is the NodePort + 30444 to 9444. A NodePort Service gives every port it lists a node port. +- **The certificate command writes the audit kubeconfig.** `generate-audit-webhook-kubeconfig.sh` + defaults to the product's names and reads Secrets; requirement 5 keeps the lab off it. +- **The webhook keeps the name `validate-all.configbutler.ai`.** The API server keys the admission + recorder's audit annotation by it (`validate-all.configbutler.ai/scenario`) and quotes it in row + 12's rejection, so the name is part of the corpus, like the match rules of requirement 3. +- **The checks are tests in the driver.** `TestSourcesLive` is the five-source check: + `lab-bootstrap` runs it, and `lab-capture` skips it. `TestClusterProvenance` writes `CLUSTER.md` + on an update and fails a capture when the cluster serves another version. The static check is + [`selfcontained_test.go`](../../test/mutationlab/selfcontained_test.go), in `task test`. +- **The capture on v1.36.4 differs in one field.** 19 of 21 scenarios reproduced the corpus byte for + byte. The other two, rows 8a and 19, gained `authenticationMetadata.impersonationConstraint: + impersonate:user-info` on each of their 15 impersonated audit events, and on nothing else. A plain + k3s v1.36.1 and v1.36.4 cluster with an audit log backend and none of the lab's wiring records the + same field for the same request, so it comes from the API server (constrained impersonation, beta + in 1.36), not from the new delivery path. Why the committed rows, captured on the e2e cluster, + lack it is not known. +- **Memory.** The lab cluster's node used about 600 MiB with the lab and the sample API server + running, so `lab-capture` does not take the e2e lock. +- **CI.** [`mutation-lab.yml`](../../.github/workflows/mutation-lab.yml) runs `lab-capture` nightly + and on manual dispatch. diff --git a/docs/proud.md b/docs/proud.md index c49fc1bef..6edbf8dad 100644 --- a/docs/proud.md +++ b/docs/proud.md @@ -96,7 +96,8 @@ be **SSH-signed**. It even captures resources served by an **aggregated API serv - A **self-ratcheting coverage gate** (`cover-check`) that only ever moves the floor up, with merged unit + e2e coverage reported to Codecov. - **Three install modes** — config-dir, Helm, and plain manifests — all exercised by e2e. -- A **mutation-capture lab** (`lab-e2e`) for stress-testing capture correctness. +- A **mutation-capture lab** (`lab-capture`), on a cluster of its own, that records what Kubernetes + emits and diffs it against a committed corpus. - `actionlint` on the CI workflows, Conventional Commits driving `release-please`. - A genuinely deep [docs/architecture.md](architecture.md) that answers the "but how does it actually…" questions instead of hand-waving. diff --git a/docs/spec/mutation-capture-lab-design.md b/docs/spec/mutation-capture-lab-design.md index 05d92bcf1..9aa9d9c64 100644 --- a/docs/spec/mutation-capture-lab-design.md +++ b/docs/spec/mutation-capture-lab-design.md @@ -13,9 +13,10 @@ The corpus is captured against **k8s v1.36.4+k3s1** (recorded in the [Difficult Cases Catalog](#difficult-cases-catalog)), plus the rows that extend it, which `test/mutationlab/README.md` indexes. -The lab reuses the product's webhook URLs (`/validate-all`, `/audit-webhook`) on the same ports and -TLS mounts, so capturing is just an **image swap** on an already-prepared e2e cluster — no new audit -policy, webhook config, or certificates. The binary lives under `cmd/mutation-capture-lab/`, the +The lab runs on a k3d cluster of its own, which delivers audit, admission, conversion, and the +aggregated API to it through their real mechanisms +([design](../design/mutation-capture-lab-own-cluster.md)). The binary lives under +`cmd/mutation-capture-lab/`, the recorders/normalizer/store/ record model under `internal/mutationlab/`, and the live-cluster driver plus golden corpus under `test/mutationlab/` (driver behind the `mutationlab_e2e` build tag, so the normal lane never needs a @@ -386,13 +387,14 @@ sometimes itself the behavior under test. ### Golden workflow -- **Default run:** capture → normalize → compare against the committed file; a mismatch fails the test - and prints a unified diff. -- **Update run:** `MUTATIONLAB_UPDATE=1 task lab-e2e` (or `task lab-corpus-update`) rewrites the corpus - from the live capture. -- **Validating a new Kubernetes version:** bump the k3d image, run the update, and review the `corpus/` - diff — it *is* the behavioral changelog for the upgrade, scoped to exactly the behaviors GitOps - Reverser depends on, with volatile fields normalized away. An empty diff is a positive result. +- **Default run:** `task lab-capture`: capture → normalize → compare against the committed file; a + mismatch fails the test and prints a unified diff. +- **Update run:** `task lab-corpus-update` rewrites the corpus from the live capture, and the k3s + image, server version, and date in `CLUSTER.md`. +- **Validating a new Kubernetes version:** recreate the lab cluster with + `task lab-bootstrap LAB_K3S_IMAGE=`, run the update, and review the `corpus/` diff — it + *is* the behavioral changelog for the upgrade, scoped to exactly the behaviors GitOps Reverser + depends on, with volatile fields normalized away. An empty diff is a positive result. ## Record model @@ -438,13 +440,17 @@ not) after the request — never guessed from the payload alone. The lab gets its own everything and reuses only the integration *shape* of the real install: - **Separate binary** (`cmd/mutation-capture-lab/main.go`), built as its own image. -- **Swap the image, reuse the wiring.** Rather than authoring its own audit policy, webhook configs, - and certificates, the lab serves the **same** webhook URLs as the product on the same ports and TLS - mounts, so `task lab-e2e` integrates by swapping the controller image on the already-prepared e2e - cluster (`test/mutationlab/swap-image.sh`). The cluster is left running the lab image; restore with - `task clean-cluster && task test-e2e`. -- **Separate, opt-in, serial task targets** (`task lab-e2e`, `task lab-corpus-update`) — not invoked by - `task test-e2e` or the default CI lane; the driver is behind the `mutationlab_e2e` build tag. +- **Its own cluster.** The lab runs on a k3d cluster of its own, `gitops-reverser-capture-lab`, + with its own certificates, `ValidatingWebhookConfiguration`, Service, and RBAC + (`test/mutationlab/deploy/`), and nothing of the product. The API server's audit webhook backend, + the admission registration, the Widget CRD's conversion webhook, and the `wardle` aggregated API + deliver to it as they would on any cluster; it shares only the cluster script, the audit policy, + and the sample API server's manifests with e2e. See + [the mutation-capture lab on its own cluster](../design/mutation-capture-lab-own-cluster.md). +- **Separate, opt-in, serial tasks** (`task lab-bootstrap`, `task lab-capture`, + `task lab-corpus-update`) in `test/mutationlab/Taskfile-lab.yml`, not invoked by `task test-e2e` + or the per-PR CI lanes; a scheduled workflow runs `lab-capture`. The driver is behind the + `mutationlab_e2e` build tag. Because audit delivery is asynchronous and batched, a late event from one scenario can land after the next has started. The lab **isolates** scenarios rather than trusting a clean slate: @@ -502,7 +508,7 @@ so each row remains a deliberate stop-and-read point. Write Git commits · reuse the GitOps Reverser controller runtime · implement the full `WatchRule`/`GitTarget` model · solve HA/retention/multi-cluster · prove every resource behaves identically · build a **mutating** admission recorder (cited, not re-proven) · re-prove **Cite**-half -contracts · hand-author corpus files · wire the lab into `task test-e2e` or the default CI lane · add +contracts · hand-author corpus files · wire the lab into `task test-e2e` or the per-PR CI lanes · add lab-only behavior to the Helm chart. ## Related diff --git a/docs/tasks-overview.md b/docs/tasks-overview.md index 36d042180..02c23058f 100644 --- a/docs/tasks-overview.md +++ b/docs/tasks-overview.md @@ -311,9 +311,13 @@ flowchart LR FULL --> ALLURE["allure-e2e-results/report/open"] FULL --> COV["e2e-coverage-collect
(when E2E_COVERAGE=1)"] - PREP --> LAB["lab-e2e / lab-corpus-update
(serial opt-in)"] ``` +The mutation-capture lab is not in this graph: it runs on a cluster of its own, through +`task lab-bootstrap`, `lab-capture`, and `lab-corpus-update` +([`test/mutationlab/Taskfile-lab.yml`](../test/mutationlab/Taskfile-lab.yml)), and no task here +depends on it or it on them. + ## Why the DAG pays off: only what changed re-runs Because each task declares its real inputs and outputs, Task skips any step already up to diff --git a/hack/e2e/lab-port-forward.sh b/hack/e2e/lab-port-forward.sh deleted file mode 100755 index 7fc427f90..000000000 --- a/hack/e2e/lab-port-forward.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Establish a robust port-forward to the mutation-capture lab's records API -# (the controller pod's :8081), mirroring hack/e2e/setup-port-forwards.sh: the -# forward is detached with setsid so it survives this shell, verified for both -# liveness and a healthy /healthz, and reused on the fast path when already up. -# -# The lab API is not on the controller Service (which exposes only audit/webhook/ -# metrics), so this forwards to the Deployment, which re-resolves to the current -# pod — important because each image swap rolls a new pod. -# -# Inputs (env): -# - CTX / E2E_KUBECONTEXT (required): kube context -# - NAMESPACE (required): controller namespace -# - LAB_API_PORT (optional, default 18081): local port -# - CONTROLLER_DEPLOY_SELECTOR (optional): label selector for the controller -# - REMOTE_PORT (optional, default 8081): the lab API port in the pod - -KUBE_CONTEXT="${E2E_KUBECONTEXT:-${CTX:-}}" -NAMESPACE="${NAMESPACE:?NAMESPACE is required}" -LAB_API_PORT="${LAB_API_PORT:-18081}" -SELECTOR="${CONTROLLER_DEPLOY_SELECTOR:-app.kubernetes.io/part-of=gitops-reverser}" -REMOTE_PORT="${REMOTE_PORT:-8081}" - -if [[ -z "${KUBE_CONTEXT}" ]]; then - KUBE_CONTEXT="$(kubectl config current-context 2>/dev/null || true)" -fi -[[ -n "${KUBE_CONTEXT}" ]] || { echo "❌ kube context is required (set CTX/E2E_KUBECONTEXT)"; exit 1; } - -healthy() { curl -fsS "http://127.0.0.1:${LAB_API_PORT}/healthz" >/dev/null 2>&1; } - -# No fast-path reuse: the lab pod is rolled by every image swap, so a surviving -# forward from a previous run points at the now-terminated old pod. Always tear -# down any existing forward and re-establish against the current ready pod. -echo "🧹 Cleaning up any stale lab port-forward on ${LAB_API_PORT}…" -pkill -f "kubectl.*port-forward.*${LAB_API_PORT}:${REMOTE_PORT}" 2>/dev/null || true -sleep 1 - -# The Deployment carries SELECTOR, but its Pods do not — derive the pod selector -# from the Deployment's own .spec.selector.matchLabels. -deploy="$(kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" get deploy -l "${SELECTOR}" -o name | head -n1)" -[[ -n "${deploy}" ]] || { echo "❌ no controller Deployment matching ${SELECTOR}"; exit 1; } -pod_selector="$(kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" get "${deploy}" \ - -o go-template='{{range $k,$v := .spec.selector.matchLabels}}{{$k}}={{$v}},{{end}}')" -pod_selector="${pod_selector%,}" -[[ -n "${pod_selector}" ]] || { echo "❌ ${deploy} has no pod selector"; exit 1; } - -echo "⏳ Waiting for the lab pod to be ready (selector ${pod_selector})…" -deadline=$((SECONDS + 120)) -pod="" -while (( SECONDS < deadline )); do - pod="$(kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" get pods -l "${pod_selector}" \ - -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' \ - | head -n1)" - if [[ -n "${pod}" ]] && kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" \ - wait --for=condition=ready "pod/${pod}" --timeout=15s >/dev/null 2>&1; then - break - fi - pod="" - sleep 2 -done -[[ -n "${pod}" ]] || { echo "❌ no ready lab pod matching ${pod_selector}"; exit 1; } - -echo "🔌 Starting lab API port-forward (${LAB_API_PORT} -> ${REMOTE_PORT})…" -pf_log="/tmp/lab-api-pf.log" -setsid kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" port-forward --address 127.0.0.1 \ - "pod/${pod}" "${LAB_API_PORT}:${REMOTE_PORT}" >"${pf_log}" 2>&1 < /dev/null & -pf_pid=$! - -for i in {1..15}; do - if ! kill -0 "${pf_pid}" 2>/dev/null; then - echo "❌ lab port-forward process died:"; sed 's/^/ /' "${pf_log}" 2>/dev/null || true - exit 1 - fi - if healthy; then - echo "✅ lab API port-forward verified on http://127.0.0.1:${LAB_API_PORT}" - exit 0 - fi - sleep 1 -done - -echo "❌ lab API port-forward did not become healthy after 15s:"; sed 's/^/ /' "${pf_log}" 2>/dev/null || true -exit 1 diff --git a/internal/mutationlab/labserver/api.go b/internal/mutationlab/labserver/api.go index bb5d75a45..9111feb34 100644 --- a/internal/mutationlab/labserver/api.go +++ b/internal/mutationlab/labserver/api.go @@ -47,8 +47,7 @@ func (a *API) Routes() *http.ServeMux { mux.HandleFunc("/records", a.handleRecords) mux.HandleFunc("/watch-probe", a.handleWatchProbe) mux.HandleFunc("/healthz", a.handleHealthz) - // /readyz mirrors /healthz so a lab deployment can satisfy the product - // deployment's readiness probe unchanged when it swaps in the lab image. + // /readyz mirrors /healthz; the lab Deployment's readiness probe reads it. mux.HandleFunc("/readyz", a.handleHealthz) return mux } diff --git a/test/e2e/Taskfile-e2e.yml b/test/e2e/Taskfile-e2e.yml index f30e5a509..aeed265ed 100644 --- a/test/e2e/Taskfile-e2e.yml +++ b/test/e2e/Taskfile-e2e.yml @@ -31,14 +31,6 @@ vars: CONTAINER_TOOL: '{{.CONTAINER_TOOL | default "docker"}}' TASK_BIN: '{{.TASK_BIN | default "task"}}' E2E_LOCAL_IMAGE: '{{.E2E_LOCAL_IMAGE | default "gitops-reverser:e2e-local"}}' - # Mutation-capture lab (opt-in, serial; see the lab-* tasks at the end of file). - # The image tag is a content hash of the lab sources, so each change produces a - # unique tag. The swap patch then changes the deployment's image ref and rolls - # out naturally; an unchanged source reuses the same tag (no needless rollout). - LAB_IMAGE_HASH: - sh: cat $(find cmd/mutation-capture-lab internal/mutationlab -type f -name '*.go' | sort) go.mod go.sum test/mutationlab/Dockerfile 2>/dev/null | sha256sum | cut -c1-12 - LAB_IMAGE: '{{.LAB_IMAGE | default (printf "mutation-capture-lab:e2e-%s" .LAB_IMAGE_HASH)}}' - LAB_API_PORT: '{{.LAB_API_PORT | default "18081"}}' PROJECT_IMAGE: sh: | if [ -n "${PROJECT_IMAGE:-}" ]; then @@ -1571,66 +1563,3 @@ tasks: kubectl --context "{{.CTX}}" apply -k "{{.DEMO_MANIFESTS_DIR}}" touch "{{.CS}}/demo.ready" - # ---------------------------------------------------------------------------- - # Mutation-capture lab (docs/spec/mutation-capture-lab-design.md). - # - # Opt-in and SERIAL. These targets are NOT invoked by `task test-e2e` and are - # NOT part of the default CI lane. The lab deliberately reuses the product's - # audit + admission webhook wiring (same URLs, same TLS cert mounts), so it just - # swaps the controller image for the lab image on the already-prepared cluster — - # no new audit policy, webhook config, or certificates. After running the lab, - # restore the product with `task clean-cluster` then `task test-e2e`. - # ---------------------------------------------------------------------------- - lab-build-image: - desc: Build the mutation-capture lab image (separate from the product image). - cmds: - - '{{.CONTAINER_TOOL}} build -f test/mutationlab/Dockerfile -t {{.LAB_IMAGE}} .' - - _lab-deployed: - # Reuse the product bring-up + install (same cluster, same audit + admission - # wiring), then swap in the lab image. Depends on _controller-deployed so the - # cluster, CRDs, certs, audit policy, and webhook configs are all in place. - deps: - - _controller-deployed - - lab-build-image - cmds: - - | - CTX="{{.CTX}}" CLUSTER_NAME="{{.CLUSTER_NAME}}" NAMESPACE="{{.NAMESPACE}}" \ - LAB_IMAGE="{{.LAB_IMAGE}}" K3D="{{.K3D}}" KUBECTL="{{.KUBECTL}}" \ - CONTROLLER_DEPLOY_SELECTOR="{{.CONTROLLER_DEPLOY_SELECTOR}}" \ - CONTROLLER_CONTAINER="{{.CONTROLLER_CONTAINER}}" \ - bash test/mutationlab/swap-image.sh - - lab-e2e: - desc: Run the mutation-capture lab end-to-end (serial). Swaps the lab image onto the cluster. - deps: - - _lab-deployed - cmds: - - task: _lab-run-driver - - lab-corpus-update: - desc: Regenerate the lab corpus from a live capture (serial, MUTATIONLAB_UPDATE=1). - deps: - - _lab-deployed - cmds: - - task: _lab-run-driver - vars: - MUTATIONLAB_UPDATE: '1' - - _lab-run-driver: - vars: - MUTATIONLAB_UPDATE: '{{.MUTATIONLAB_UPDATE | default ""}}' - cmds: - - | - set -euo pipefail - # Robust, detached port-forward to the lab API, reusing the established - # e2e port-forward pattern (setsid + ready-pod wait + health verify). The - # driver additionally retries transient connection errors. - CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" LAB_API_PORT="{{.LAB_API_PORT}}" \ - CONTROLLER_DEPLOY_SELECTOR="{{.CONTROLLER_DEPLOY_SELECTOR}}" \ - bash hack/e2e/lab-port-forward.sh - # Serial by construction: a single `go test` package run, no Ginkgo --procs. - LAB_API_URL="http://127.0.0.1:{{.LAB_API_PORT}}" \ - LAB_KUBECONTEXT="{{.CTX}}" \ - MUTATIONLAB_UPDATE="{{.MUTATIONLAB_UPDATE}}" \ - go test -tags mutationlab_e2e -count=1 -timeout 10m -v ./test/mutationlab/e2e/ diff --git a/test/e2e/cluster/README.md b/test/e2e/cluster/README.md index 6f7360c73..68e205bf0 100644 --- a/test/e2e/cluster/README.md +++ b/test/e2e/cluster/README.md @@ -31,6 +31,10 @@ audit webhook setup restarts the server container to load the final webhook conf keeps that restart from taking the manager, Valkey and Gitea down with it. `K3D_AGENT_COUNT=0` still builds a single-node cluster, with no taint. +`K3D_SWITCH_CONTEXT=false` creates the cluster without making it the kubeconfig's current context; +the mutation-capture lab's own cluster ([`test/mutationlab/Taskfile-lab.yml`](../../mutationlab/Taskfile-lab.yml)) +is created that way, so bootstrapping it never moves `kubectl`, or an e2e run, onto it. + It also disables these packaged k3s components by default: - `traefik` diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh index e62ddb410..ff4d18d8b 100644 --- a/test/e2e/cluster/start-cluster.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -16,6 +16,9 @@ KUBE_APISERVER_MAX_MUTATING_REQUESTS_INFLIGHT="${KUBE_APISERVER_MAX_MUTATING_REQ # accepted for a single-node cluster, which leaves the server untainted. K3D_AGENT_COUNT="${K3D_AGENT_COUNT:-1}" K3S_IMAGE="${K3S_IMAGE:-rancher/k3s:v1.37.1-k3s1}" +# false leaves the kubeconfig's current context where it was, so a second cluster (the +# mutation-capture lab) can be created without moving kubectl, or an e2e run, onto it. +K3D_SWITCH_CONTEXT="${K3D_SWITCH_CONTEXT:-true}" AUDIT_DIR_REL="${AUDIT_DIR_REL:-test/e2e/cluster/audit}" K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" REPO_PWD="$(pwd -P)" @@ -265,7 +268,7 @@ create_cluster() { --servers 1 --agents "${K3D_AGENT_COUNT}" --kubeconfig-update-default - --kubeconfig-switch-context + "--kubeconfig-switch-context=${K3D_SWITCH_CONTEXT}" -v "${audit_host_dir}:/etc/kubernetes/audit@server:0" ) if [ -n "${K3D_API_PORT}" ]; then @@ -280,17 +283,20 @@ create_cluster() { } merge_kubeconfig() { - # Ensure kubeconfig is present and current-context points at this cluster + # Ensure kubeconfig holds this cluster's context, and is current unless K3D_SWITCH_CONTEXT=false k3d kubeconfig merge "${CLUSTER_NAME}" \ - --kubeconfig-switch-context \ + "--kubeconfig-switch-context=${K3D_SWITCH_CONTEXT}" \ --kubeconfig-merge-default >/dev/null } rewrite_kubeconfig_for_devcontainer() { - local cluster_entry server host port + local context_name cluster_entry server host port - cluster_entry="$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')" - server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" + # The named context, not the current one: with K3D_SWITCH_CONTEXT=false the current + # context is some other cluster, whose endpoint this must not touch. + context_name="$(cluster_context_name)" + cluster_entry="$(kubectl config view --context "${context_name}" --minify -o jsonpath='{.clusters[0].name}')" + server="$(kubectl config view --context "${context_name}" --minify -o jsonpath='{.clusters[0].cluster.server}')" # Parse https://HOST:PORT if [[ "$server" =~ ^https://([^:/]+):([0-9]+)$ ]]; then diff --git a/test/e2e/setup/manifests/kustomization.yaml b/test/e2e/setup/manifests/kustomization.yaml index b2d215899..4e1757983 100644 --- a/test/e2e/setup/manifests/kustomization.yaml +++ b/test/e2e/setup/manifests/kustomization.yaml @@ -5,12 +5,4 @@ resources: - prometheus/instance.yaml - prometheus/prometheus-rbac.yaml - prometheus/servicemonitor.yaml - - sample-apiserver/namespace.yaml - - sample-apiserver/sa.yaml - - sample-apiserver/deployment.yaml - - sample-apiserver/service.yaml - - sample-apiserver/apiservice.yaml - - sample-apiserver/auth-delegator.yaml - - sample-apiserver/auth-reader.yaml - - sample-apiserver/rbac.yaml - - sample-apiserver/rbac-bind.yaml + - sample-apiserver diff --git a/test/e2e/setup/manifests/sample-apiserver/kustomization.yaml b/test/e2e/setup/manifests/sample-apiserver/kustomization.yaml new file mode 100644 index 000000000..2b5fdcc7a --- /dev/null +++ b/test/e2e/setup/manifests/sample-apiserver/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The wardle sample aggregated API server. The e2e setup and the mutation-capture lab +# (test/mutationlab/Taskfile-lab.yml) both apply this directory, so the two clusters +# serve the same aggregated API. +resources: + - namespace.yaml + - sa.yaml + - deployment.yaml + - service.yaml + - apiservice.yaml + - auth-delegator.yaml + - auth-reader.yaml + - rbac.yaml + - rbac-bind.yaml diff --git a/test/mutationlab/Dockerfile b/test/mutationlab/Dockerfile index 12e0315b0..859fee1c5 100644 --- a/test/mutationlab/Dockerfile +++ b/test/mutationlab/Dockerfile @@ -1,5 +1,5 @@ -# Mutation-capture lab image. Built and deployed only by `task lab-e2e`; it is -# never part of the product image, the Helm chart, or the default CI lane. +# Mutation-capture lab image. Built by `task lab-build-image` and run on the lab's +# own cluster; it is never part of the product image or the Helm chart. FROM golang:1.27.3 AS builder ARG TARGETOS diff --git a/test/mutationlab/README.md b/test/mutationlab/README.md index 73c3c6bdd..e42feddcc 100644 --- a/test/mutationlab/README.md +++ b/test/mutationlab/README.md @@ -12,7 +12,10 @@ GitOps Reverser — see [the design](../../docs/spec/mutation-capture-lab-design | `cmd/mutation-capture-lab/` | the lab binary (recorders + records API) | | `internal/mutationlab/` | record model, normalizer, store, corpus, recorders, API | | `test/mutationlab/Dockerfile` | the lab image (built only by `task lab-build-image`) | -| `test/mutationlab/swap-image.sh` | swaps the lab image onto the controller Deployment | +| `test/mutationlab/Taskfile-lab.yml` | the lab's tasks: its own cluster, deployment, and capture | +| `test/mutationlab/deploy/` | the lab's namespace, RBAC, Deployment, Services, and webhook registration | +| `test/mutationlab/tools/lab-certs/` | issues the lab's CA, serving certificate, and the API server's audit client certificate | +| `test/mutationlab/deploy.sh`, `port-forward.sh` | deploy onto the lab cluster; forward its records API | | `test/mutationlab/e2e/` | the live-cluster driver (build tag `mutationlab_e2e`) | | `test/mutationlab/corpus/` | the committed golden corpus + `CLUSTER.md` provenance | @@ -69,29 +72,38 @@ transport-only events can be scenario-attributed — see the [watch-first ingestion architecture](../../docs/finished/watch-first-ingestion-architecture.md) design notes. -> ⚠️ **The "no audit" rows record this cluster's audit POLICY, not the API server.** The lab runs -> against the already-prepared e2e cluster and reuses its policy +> ⚠️ **The "no audit" rows record this cluster's audit POLICY, not the API server.** The lab cluster +> uses the e2e audit policy > ([`test/e2e/cluster/audit/policy.yaml`](../e2e/cluster/audit/policy.yaml)), which drops `pods` and > every `*/status` at `level: None` as runtime noise. So rows 5 and 7 show what a cluster configured > like this one does not tell us — a `DELETE` on a pod is an audited request like any other when the > policy asks for it. Read them as "excluded here", not as "unknowable". Re-capturing either row > against a policy that includes those types would settle it by measurement. -## How it integrates: swap the image, reuse the wiring +## How it integrates: a cluster of its own -The lab serves the **same** webhook URLs as the product — -`/validate-all` and `/audit-webhook` — on the same ports and TLS cert mounts. So -making a cluster capture with the lab is just swapping the controller image: no -new audit policy, webhook config, or certificates. `task lab-e2e` does this on the -already-prepared e2e cluster, then drives the scenarios serially. +The lab runs on its own k3d cluster, `gitops-reverser-capture-lab`, which runs the lab, the +`wardle` sample aggregated API server, and nothing of the product +([design](../../docs/design/mutation-capture-lab-own-cluster.md)). Every source arrives the way it +does on any cluster: -Reusing the wiring means reusing whatever the cluster's audit kubeconfig points at, and -this one names its route: it posts to `/audit-webhook/gitops-reverser.local`, the route of the -`local` KubeProvider in `gitops-reverser`, because the bare path is the product's shared annotation-routed -endpoint. The lab therefore serves the whole `/audit-webhook/` subtree and records what -arrives whichever source it was addressed to. Serving only the exact bare path 404s every -event the cluster sends — which reads as a lab that captures watches and admission but -never a single audit record, in every scenario at once. +- the API server's **audit webhook backend** posts to `https://127.0.0.1:30444/audit-webhook`, a + NodePort on its own node, over mutual TLS: it presents a client certificate from the lab CA, and + the lab refuses a connection without one; +- the `ValidatingWebhookConfiguration` `mutation-capture-lab` sends **admission** reviews to the + lab's Service; +- the scenario Widget CRD's **conversion** webhook calls the same Service at `/convert`; +- the lab **watches** through its own ServiceAccount, with read-only RBAC on what it watches. + +`lab-certs` issues the certificates before the cluster exists, so the audit kubeconfig the API +server reads at startup is already the final one and the cluster never restarts. The cluster is +created by the same [`start-cluster.sh`](../e2e/cluster/start-cluster.sh) as the e2e cluster, with +the same audit flags and policy, on one node, and without switching the current kube context. + +Nothing in the lab's wiring names a product resource or depends on a product task; +[`selfcontained_test.go`](selfcontained_test.go) enforces that in `task test`. The lab used to swap +its image into the product's Deployment and borrow its webhook registration and certificates, and +lost its admission source the day the product removed them, with every check still green. Row 15 (aggregated API write) is what settled the body-enrichment question: the official audit event for an aggregated-API write carries an empty body, yet the @@ -101,28 +113,35 @@ retired — so the lab no longer serves or records that endpoint. ## Running it (opt-in, serial) -These targets are **not** part of `task test-e2e` or the default CI lane. +These tasks are not part of `task test-e2e` or the per-PR CI lanes; the +[mutation lab workflow](../../.github/workflows/mutation-lab.yml) runs `lab-capture` nightly. ```bash -# Prepare the e2e cluster + product wiring, swap in the lab image, capture, diff: -task lab-e2e - -# Accept a new/changed capture as the golden corpus: -task lab-corpus-update +task lab-bootstrap # certificates, cluster, image, deployment, and a live check of all five sources +task lab-capture # run the scenarios and compare the capture to the corpus +task lab-corpus-update # accept a new or changed capture as the corpus, CLUSTER.md included +task lab-status # the cluster, its k3s version, the lab pod, and how many records it holds +task lab-cleanup # delete the cluster and its stamps ``` -`task lab-e2e` leaves the cluster running the lab image. To restore the product: - -```bash -task clean-cluster && task test-e2e -``` +The lab cluster can run beside the e2e cluster: its own name, stamps +(`.stamps/cluster/k3d-gitops-reverser-capture-lab/`), run lock, and records-API port (18081). It +needs no restore afterwards; the e2e cluster is never touched. The unit tests for the lab packages run in the normal lane (`task test`); the -`test/mutationlab/e2e/` driver is behind the `mutationlab_e2e` build tag and only -runs under `task lab-e2e`. +`test/mutationlab/e2e/` driver is behind the `mutationlab_e2e` build tag and runs only through the +lab's tasks. ## Validating a new Kubernetes version -Bump the k3d image, run `task lab-corpus-update`, and review the `corpus/` diff: -it is the behavioral changelog for the upgrade, scoped to exactly the fine-grained -behaviors GitOps Reverser depends on. An empty diff is a positive result. +```bash +task lab-cleanup +task lab-bootstrap LAB_K3S_IMAGE=rancher/k3s:v1.38.0-k3s1 +task lab-corpus-update +git diff test/mutationlab/corpus/ +``` + +The `corpus/` diff is the behavioral changelog for the upgrade, scoped to exactly the fine-grained +behaviors GitOps Reverser depends on. An empty diff is a positive result. Without `LAB_K3S_IMAGE`, a +new lab cluster runs the k3s image `CLUSTER.md` records, and `lab-capture` fails when the cluster +serves another version than the corpus was captured on. diff --git a/test/mutationlab/Taskfile-lab.yml b/test/mutationlab/Taskfile-lab.yml new file mode 100644 index 000000000..1672dc747 --- /dev/null +++ b/test/mutationlab/Taskfile-lab.yml @@ -0,0 +1,211 @@ +# yaml-language-server: $schema=https://taskfile.dev/schema.json +version: '3' + +# The mutation-capture lab on its own k3d cluster +# (docs/design/mutation-capture-lab-own-cluster.md). It runs the lab and nothing of the product, +# beside the e2e cluster if one is up: its own cluster name, stamps, lock, and port, and it never +# switches the current kube context. +# +# Self-contained by rule: no task here depends on a task outside this file, and nothing here names +# a product resource. test/mutationlab/selfcontained_test.go enforces both in `task test`. +# +# task lab-bootstrap certificates, cluster, image, deployment, and a live check of every source +# task lab-capture run the scenarios and compare the capture to the corpus +# task lab-corpus-update run the scenarios and write the capture as the corpus, CLUSTER.md included +# task lab-status the cluster, its k3s version, the lab pod, and how many records it holds +# task lab-cleanup delete the cluster and its stamp directory +# +# Testing a new Kubernetes version: +# task lab-cleanup && task lab-bootstrap LAB_K3S_IMAGE=rancher/k3s: && task lab-corpus-update + +vars: + LAB_CLUSTER_NAME: '{{.LAB_CLUSTER_NAME | default "gitops-reverser-capture-lab"}}' + LAB_CTX: 'k3d-{{.LAB_CLUSTER_NAME}}' + LAB_CS: '.stamps/cluster/{{.LAB_CTX}}' + LAB_NAMESPACE: mutation-capture-lab + LAB_API_PORT: '{{.LAB_API_PORT | default "18081"}}' + LAB_CONTAINER_TOOL: '{{.LAB_CONTAINER_TOOL | default "docker"}}' + LAB_AUDIT_POLICY_SOURCE: test/e2e/cluster/audit/policy.yaml + # The k3s image a new lab cluster is created with: LAB_K3S_IMAGE when given, else the one the + # committed corpus was captured on, as test/mutationlab/corpus/CLUSTER.md records it. An + # existing cluster keeps the image it was created with, recorded in its stamps. + LAB_K3S_IMAGE_RECORDED: + sh: sed -n 's/^| k3d image | `\([^`]*\)` |$/\1/p' test/mutationlab/corpus/CLUSTER.md + LAB_K3S_IMAGE_REQUESTED: '{{.LAB_K3S_IMAGE | default ""}}' + # The image tag is a content hash of the lab's sources, so a change rolls the lab pod and an + # unchanged source keeps it running. + LAB_IMAGE_HASH: + sh: >- + cat $(find cmd/mutation-capture-lab internal/mutationlab -type f -name '*.go' | sort) + go.mod go.sum test/mutationlab/Dockerfile | sha256sum | cut -c1-12 + LAB_IMAGE: 'mutation-capture-lab:{{.LAB_IMAGE_HASH}}' + +tasks: + lab-bootstrap: + desc: Create the mutation-capture lab's own cluster, deploy the lab, and prove every source is live. + cmds: + - task: _lab-deployed + - task: _lab-port-forward + # The probe writes of every source, found among the records: audit, admission, conversion, + # the aggregated API, and the watch. + - task: _lab-driver + vars: + LAB_RUN: '^TestSourcesLive$' + + lab-capture: + desc: Run the lab scenarios on the lab cluster and compare the capture to the committed corpus. + cmds: + - task: lab-bootstrap + - task: _lab-driver + vars: + LAB_SKIP: '^TestSourcesLive$' + + lab-corpus-update: + desc: Run the lab scenarios and write the capture as the corpus, CLUSTER.md included. + cmds: + - task: lab-bootstrap + - task: _lab-driver + vars: + LAB_SKIP: '^TestSourcesLive$' + MUTATIONLAB_UPDATE: '1' + + lab-status: + desc: Show the lab cluster, its k3s version, the lab pod, and how many records it holds. + cmds: + - | + if ! k3d cluster list "{{.LAB_CLUSTER_NAME}}" >/dev/null 2>&1; then + echo "No lab cluster '{{.LAB_CLUSTER_NAME}}'. Create it with: task lab-bootstrap" + exit 0 + fi + k3d cluster list "{{.LAB_CLUSTER_NAME}}" + echo + echo "k3s image: $(cat "{{.LAB_CS}}/k3s-image" 2>/dev/null || echo unknown)" + kubectl --context "{{.LAB_CTX}}" version 2>/dev/null | grep -i '^server' || echo "Server: unreachable" + echo + kubectl --context "{{.LAB_CTX}}" -n "{{.LAB_NAMESPACE}}" get pods -o wide || true + echo + if records="$(curl -fsS "http://127.0.0.1:{{.LAB_API_PORT}}/records" 2>/dev/null)"; then + count="$(printf '%s' "${records}" | grep -o '"source":"[a-z]*"' | wc -l)" + echo "Records held: ${count}" + else + echo "Records held: unknown (no forward on {{.LAB_API_PORT}}; task lab-bootstrap starts one)" + fi + + lab-cleanup: + desc: Delete the lab cluster and its stamp directory. + cmds: + - pkill -f "port-forward --context {{.LAB_CTX}} " 2>/dev/null || true + # Unconditional: an interrupted create leaves state `k3d cluster list` does not show, and + # delete exits 0 when there is nothing to delete. + - k3d cluster delete "{{.LAB_CLUSTER_NAME}}" + - rm -rf "{{.LAB_CS}}" + + lab-build-image: + desc: Build the mutation-capture lab image. + status: + - '{{.LAB_CONTAINER_TOOL}} image inspect "{{.LAB_IMAGE}}" >/dev/null 2>&1' + cmds: + - '{{.LAB_CONTAINER_TOOL}} build -f test/mutationlab/Dockerfile -t "{{.LAB_IMAGE}}" .' + + _lab-certs: + # Issued before the cluster exists, so the audit kubeconfig the API server reads at startup is + # already the final one. + status: + - test -f "{{.LAB_CS}}/certs/ca.crt" + - test -f "{{.LAB_CS}}/certs/tls.crt" + - test -f "{{.LAB_CS}}/certs/tls.key" + - test -f "{{.LAB_CS}}/audit/webhook-config.yaml" + cmds: + - >- + go run ./test/mutationlab/tools/lab-certs + --cert-dir "{{.LAB_CS}}/certs" + --audit-kubeconfig "{{.LAB_CS}}/audit/webhook-config.yaml" + --namespace "{{.LAB_NAMESPACE}}" + + _lab-cluster-ready: + deps: + - _lab-certs + preconditions: + # A stamped cluster that does not answer is wedged (host suspend, OOM, a killed run); + # recreating it under the same stamps would only hide that. + - sh: | + test -f "{{.LAB_CS}}/ready" || exit 0 + kubectl --context "{{.LAB_CTX}}" --request-timeout=5s get nodes \ + -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null \ + | awk 'BEGIN{n=0;t=0} {t++; if ($0=="True") n++} END{exit !(t>0 && n==t)}' + msg: | + The lab cluster '{{.LAB_CLUSTER_NAME}}' is stamped as ready but its nodes are not Ready. + Recreate it: task lab-cleanup && task lab-bootstrap + - sh: | + test -f "{{.LAB_CS}}/ready" || exit 0 + test -z "{{.LAB_K3S_IMAGE_REQUESTED}}" || test "$(cat "{{.LAB_CS}}/k3s-image")" = "{{.LAB_K3S_IMAGE_REQUESTED}}" + msg: | + The lab cluster runs another k3s image than LAB_K3S_IMAGE={{.LAB_K3S_IMAGE_REQUESTED}}. + Recreate it on that image: task lab-cleanup && task lab-bootstrap LAB_K3S_IMAGE={{.LAB_K3S_IMAGE_REQUESTED}} + status: + - test -f "{{.LAB_CS}}/ready" + cmds: + - mkdir -p "{{.LAB_CS}}/audit" + # The one tracked audit policy, shared with e2e, so the corpus shows what the product's + # clusters audit. + - cp "{{.LAB_AUDIT_POLICY_SOURCE}}" "{{.LAB_CS}}/audit/policy.yaml" + - | + image="{{.LAB_K3S_IMAGE_REQUESTED | default .LAB_K3S_IMAGE_RECORDED}}" + test -n "${image}" || { echo "no k3s image: set LAB_K3S_IMAGE or record one in CLUSTER.md" >&2; exit 1; } + # One node: the lab never restarts its server, so it needs no agent to keep pods away + # from a restart. + CLUSTER_NAME="{{.LAB_CLUSTER_NAME}}" \ + K3S_IMAGE="${image}" \ + AUDIT_DIR_REL="{{.LAB_CS}}/audit" \ + K3D_AGENT_COUNT=0 \ + K3D_SWITCH_CONTEXT=false \ + bash test/e2e/cluster/start-cluster.sh + printf '%s\n' "${image}" > "{{.LAB_CS}}/k3s-image" + touch "{{.LAB_CS}}/ready" + + _lab-deployed: + deps: + - _lab-cluster-ready + - lab-build-image + cmds: + - | + LAB_CLUSTER_NAME="{{.LAB_CLUSTER_NAME}}" LAB_CTX="{{.LAB_CTX}}" LAB_CS="{{.LAB_CS}}" \ + LAB_IMAGE="{{.LAB_IMAGE}}" LAB_NAMESPACE="{{.LAB_NAMESPACE}}" \ + CONTAINER_TOOL="{{.LAB_CONTAINER_TOOL}}" \ + bash test/mutationlab/deploy.sh + + _lab-port-forward: + run: always + cmds: + - | + LAB_CTX="{{.LAB_CTX}}" LAB_CS="{{.LAB_CS}}" LAB_NAMESPACE="{{.LAB_NAMESPACE}}" \ + LAB_API_PORT="{{.LAB_API_PORT}}" \ + bash test/mutationlab/port-forward.sh + + _lab-driver: + # always: the root Taskfile runs each task once per invocation, and this one runs twice in a + # capture, once for the live check and once for the scenarios. + run: always + vars: + LAB_RUN: '{{.LAB_RUN | default ""}}' + LAB_SKIP: '{{.LAB_SKIP | default ""}}' + MUTATIONLAB_UPDATE: '{{.MUTATIONLAB_UPDATE | default ""}}' + cmds: + - | + mkdir -p "{{.LAB_CS}}" + # Serial by construction (one `go test` package run), and one run per cluster at a time: + # scenarios clear the shared record store. + rc=0 + LAB_API_URL="http://127.0.0.1:{{.LAB_API_PORT}}" \ + LAB_KUBECONTEXT="{{.LAB_CTX}}" \ + LAB_CA_FILE="$(pwd)/{{.LAB_CS}}/certs/ca.crt" \ + LAB_K3S_IMAGE="$(cat "{{.LAB_CS}}/k3s-image")" \ + MUTATIONLAB_UPDATE="{{.MUTATIONLAB_UPDATE}}" \ + flock --nonblock --conflict-exit-code 75 "{{.LAB_CS}}/lab.lock" \ + go test -tags mutationlab_e2e -count=1 -timeout 15m -v \ + {{if .LAB_RUN}}-run '{{.LAB_RUN}}'{{end}} {{if .LAB_SKIP}}-skip '{{.LAB_SKIP}}'{{end}} \ + ./test/mutationlab/e2e/ || rc=$? + if [ "${rc}" -eq 75 ]; then + echo "Another lab run holds {{.LAB_CS}}/lab.lock; wait for it to finish." >&2 + fi + exit "${rc}" diff --git a/test/mutationlab/deploy.sh b/test/mutationlab/deploy.sh new file mode 100644 index 000000000..435bfe097 --- /dev/null +++ b/test/mutationlab/deploy.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Deploy the mutation-capture lab and the wardle sample aggregated API server onto the lab's own +# cluster, and wait until both serve. Idempotent: an unchanged image and certificate set applies +# as a no-op. +# +# Inputs (env): +# - LAB_CLUSTER_NAME (required): k3d cluster name, without the k3d- prefix +# - LAB_CTX (required): its kube context +# - LAB_CS (required): its stamp directory, holding certs/ from test/mutationlab/tools/lab-certs +# - LAB_IMAGE (required): the lab image, already built +# - LAB_NAMESPACE (optional, default mutation-capture-lab) +# - K3D / KUBECTL / CONTAINER_TOOL (optional): tool overrides + +: "${LAB_CLUSTER_NAME:?LAB_CLUSTER_NAME is required}" +: "${LAB_CTX:?LAB_CTX is required}" +: "${LAB_CS:?LAB_CS is required}" +: "${LAB_IMAGE:?LAB_IMAGE is required}" +LAB_NAMESPACE="${LAB_NAMESPACE:-mutation-capture-lab}" +K3D="${K3D:-k3d}" +KUBECTL="${KUBECTL:-kubectl}" +CONTAINER_TOOL="${CONTAINER_TOOL:-docker}" +NODE="k3d-${LAB_CLUSTER_NAME}-server-0" +LAB_DIR="test/mutationlab" +SAMPLE_APISERVER_DIR="test/e2e/setup/manifests/sample-apiserver" + +kubectl_lab() { "${KUBECTL}" --context "${LAB_CTX}" "$@"; } + +image_in_node() { "${CONTAINER_TOOL}" exec "${NODE}" crictl inspecti "${LAB_IMAGE}" >/dev/null 2>&1; } + +# import_image delivers the lab image into the node. `-m direct` streams `docker save` straight +# into the node's `ctr image import`: the default tools-node mode can misread the save as finished +# before the tarball lands and still exit 0. Direct mode can hang instead, so each attempt is +# bounded, and every attempt is checked against the node rather than trusted. +import_image() { + if image_in_node; then + echo "✅ ${LAB_IMAGE} is already in ${NODE}" + return 0 + fi + local attempt + for attempt in 1 2 3; do + echo "📦 Importing ${LAB_IMAGE} into ${LAB_CLUSTER_NAME} (attempt ${attempt}/3)…" + timeout --kill-after=10s 300s "${K3D}" image import -m direct -c "${LAB_CLUSTER_NAME}" "${LAB_IMAGE}" || true + if image_in_node; then + return 0 + fi + done + echo "❌ ${LAB_IMAGE} did not reach ${NODE} after 3 attempts" >&2 + return 1 +} + +# write_overlay renders the overlay that binds the tracked manifests to this cluster: the image +# tag, the Secret with the issued certificates, and the CA as the registration's caBundle. It +# lives in the stamp directory beside certs/, because kustomize reads a generator's files only +# from inside its own root. +write_overlay() { + local base ca_bundle + base="$(realpath --relative-to="${LAB_CS}" "${LAB_DIR}/deploy")" + ca_bundle="$(base64 -w0 <"${LAB_CS}/certs/ca.crt")" + cat >"${LAB_CS}/kustomization.yaml" </dev/null | grep -q flunders; then + return 0 + fi + sleep 2 + done + echo "❌ the wardle APIService is Available, but discovery never listed flunders" >&2 + kubectl_lab get apiservice v1alpha1.wardle.example.com -o yaml >&2 || true + return 1 +} + +import_image +write_overlay +echo "🚀 Applying the lab and the sample aggregated API server…" +kubectl_lab apply -k "${LAB_CS}" +kubectl_lab apply -k "${SAMPLE_APISERVER_DIR}" +kubectl_lab -n "${LAB_NAMESPACE}" rollout status deployment/mutation-capture-lab --timeout=180s +wait_aggregated_api +echo "✅ The lab and the sample aggregated API server are serving." diff --git a/test/mutationlab/deploy/deployment.yaml b/test/mutationlab/deploy/deployment.yaml new file mode 100644 index 000000000..8be92f081 --- /dev/null +++ b/test/mutationlab/deploy/deployment.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mutation-capture-lab + namespace: mutation-capture-lab + labels: + app.kubernetes.io/name: mutation-capture-lab +spec: + # One replica: the records live in this pod's memory, so every audit batch, admission review, + # and watch event has to reach the same store. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: mutation-capture-lab + template: + metadata: + labels: + app.kubernetes.io/name: mutation-capture-lab + spec: + serviceAccountName: mutation-capture-lab + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: lab + # The overlay sets the content-hashed tag; the image is imported into the node, never + # pulled. + image: mutation-capture-lab + imagePullPolicy: Never + args: + - --admission-addr=:9443 + - --admission-cert-dir=/certs + - --audit-addr=:9444 + - --audit-cert-dir=/certs + # The API server presents a client certificate from the lab CA, and the audit + # server refuses a connection without one, as a production receiver does. + - --audit-client-ca=/certs/ca.crt + - --api-addr=:8080 + # configmaps (rows 1-4, 8-13, 16-19), deployments (rows 5, 6), pods (row 7), the + # two-version Widget CRD in its storage version v2 (row 14), and the wardle + # aggregated API (row 15). A watch on a type that does not exist yet retries until + # it does. + - --watch-resources=v1/configmaps,apps/v1/deployments,v1/pods,mutationlab.configbutler.ai/v2/widgets,wardle.example.com/v1alpha1/flunders + ports: + - name: webhook + containerPort: 9443 + - name: audit + containerPort: 9444 + - name: api + containerPort: 8080 + readinessProbe: + httpGet: + path: /readyz + port: api + periodSeconds: 2 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 50m + memory: 64Mi + volumeMounts: + - name: certs + mountPath: /certs + readOnly: true + volumes: + - name: certs + secret: + secretName: mutation-capture-lab-tls diff --git a/test/mutationlab/deploy/kustomization.yaml b/test/mutationlab/deploy/kustomization.yaml new file mode 100644 index 000000000..f12f99b28 --- /dev/null +++ b/test/mutationlab/deploy/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The mutation-capture lab on its own cluster (docs/design/mutation-capture-lab-own-cluster.md). +# `task lab-bootstrap` applies this through an overlay in the lab's stamp directory, which adds +# the content-hashed image tag, the mutation-capture-lab-tls Secret issued by +# test/mutationlab/tools/lab-certs, and that CA as the registration's caBundle. +resources: + - namespace.yaml + - rbac.yaml + - deployment.yaml + - service.yaml + - validating-webhook.yaml diff --git a/test/mutationlab/deploy/namespace.yaml b/test/mutationlab/deploy/namespace.yaml new file mode 100644 index 000000000..8c6992c33 --- /dev/null +++ b/test/mutationlab/deploy/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mutation-capture-lab diff --git a/test/mutationlab/deploy/rbac.yaml b/test/mutationlab/deploy/rbac.yaml new file mode 100644 index 000000000..01b4bb285 --- /dev/null +++ b/test/mutationlab/deploy/rbac.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mutation-capture-lab + namespace: mutation-capture-lab +--- +# Read-only, and only what the Deployment's --watch-resources names: the watch recorder and the +# /watch-probe endpoint open watches, nothing else. A resource added there needs a rule here. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mutation-capture-lab +rules: + - apiGroups: [""] + resources: ["configmaps", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["mutationlab.configbutler.ai"] + resources: ["widgets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["wardle.example.com"] + resources: ["flunders"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: mutation-capture-lab +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mutation-capture-lab +subjects: + - kind: ServiceAccount + name: mutation-capture-lab + namespace: mutation-capture-lab diff --git a/test/mutationlab/deploy/service.yaml b/test/mutationlab/deploy/service.yaml new file mode 100644 index 000000000..13e3ce45c --- /dev/null +++ b/test/mutationlab/deploy/service.yaml @@ -0,0 +1,35 @@ +# Admission reviews, CRD conversion, and the records API. The API server addresses the webhooks +# through this Service, so its DNS name is the one the serving certificate carries. +apiVersion: v1 +kind: Service +metadata: + name: mutation-capture-lab + namespace: mutation-capture-lab +spec: + selector: + app.kubernetes.io/name: mutation-capture-lab + ports: + - name: webhook + port: 443 + targetPort: webhook + - name: api + port: 8080 + targetPort: api +--- +# Audit. The audit webhook backend is configured before the cluster exists, when no Service has a +# ClusterIP yet, so the API server posts to this fixed NodePort on its own node: +# https://127.0.0.1:30444/audit-webhook, verifying the Service's DNS name as tls-server-name. +apiVersion: v1 +kind: Service +metadata: + name: mutation-capture-lab-audit + namespace: mutation-capture-lab +spec: + type: NodePort + selector: + app.kubernetes.io/name: mutation-capture-lab + ports: + - name: audit + port: 9444 + targetPort: audit + nodePort: 30444 diff --git a/test/mutationlab/deploy/validating-webhook.yaml b/test/mutationlab/deploy/validating-webhook.yaml new file mode 100644 index 000000000..0cd484133 --- /dev/null +++ b/test/mutationlab/deploy/validating-webhook.yaml @@ -0,0 +1,36 @@ +# The rules the corpus was captured under; changing them changes what the corpus records. +# +# - resources: ["*"] matches top-level resources only, so rows 5 and 6 record no admission for +# /status and /scale. Kubernetes would send those reviews if a rule named the subresources. +# - failurePolicy Ignore: the backend is a pod on this cluster, and a Fail policy on "*" would +# refuse the writes that bring that pod back. +# - timeoutSeconds 1: with Ignore, a longer wait buys nothing but latency on every write. +# - The webhook's name is part of the capture too: the API server keys the audit annotation the +# admission recorder returns by it (validate-all.configbutler.ai/scenario), and quotes it in a +# rejection's message (row 12). +# +# The overlay `task lab-bootstrap` writes sets caBundle to the lab CA. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: mutation-capture-lab +webhooks: + - name: validate-all.configbutler.ai + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: mutation-capture-lab + namespace: mutation-capture-lab + path: /validate-all + port: 443 + caBundle: "" + failurePolicy: Ignore + matchPolicy: Equivalent + rules: + - apiGroups: ["*"] + apiVersions: ["*"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["*"] + scope: "*" + sideEffects: None + timeoutSeconds: 1 diff --git a/test/mutationlab/e2e/cluster_provenance_test.go b/test/mutationlab/e2e/cluster_provenance_test.go new file mode 100644 index 000000000..cf61f8510 --- /dev/null +++ b/test/mutationlab/e2e/cluster_provenance_test.go @@ -0,0 +1,110 @@ +//go:build mutationlab_e2e + +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "k8s.io/client-go/kubernetes" +) + +// The rows of test/mutationlab/corpus/CLUSTER.md this test owns. The prose around the table is +// left alone. +var ( + imageRow = regexp.MustCompile("(?m)^\\| k3d image \\| `([^`]*)` \\|$") + versionRow = regexp.MustCompile("(?m)^\\| Server version \\| `([^`]*)` \\(([^)]*)\\) \\|$") + capturedRow = regexp.MustCompile(`(?m)^\| Captured at \| ([^|]*) \|$`) +) + +// TestClusterProvenance keeps test/mutationlab/corpus/CLUSTER.md and the cluster that produced the +// corpus in agreement. With MUTATIONLAB_UPDATE it records the cluster's k3s image, server version, +// and the date; otherwise it fails when the cluster runs another version than the corpus was +// captured on, because a corpus diff is then a Kubernetes change, and is accepted with `task +// lab-corpus-update`, not read as a regression. +func TestClusterProvenance(t *testing.T) { + if os.Getenv("LAB_API_URL") == "" { + t.Skip("LAB_API_URL not set; run via `task lab-capture`") + } + image := os.Getenv("LAB_K3S_IMAGE") + if image == "" { + t.Fatal("LAB_K3S_IMAGE not set; run via `task lab-capture`") + } + cfg, err := restConfig() + if err != nil { + t.Fatalf("kube config: %v", err) + } + kube, err := kubernetes.NewForConfig(cfg) + if err != nil { + t.Fatalf("kube client: %v", err) + } + info, err := kube.Discovery().ServerVersion() + if err != nil { + t.Fatalf("server version: %v", err) + } + // rancher/k3s:v1.36.4-k3s1 serves v1.36.4+k3s1. A mismatch means the image recorded in the + // stamps is not what the cluster runs, and writing either into CLUSTER.md would be wrong. + if want := k3sServerVersion(image); want != info.GitVersion { + t.Fatalf("the lab cluster serves %s, but its stamps say it was created from %s", info.GitVersion, image) + } + + path := filepath.Join("..", "corpus", "CLUSTER.md") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + doc := string(raw) + if os.Getenv("MUTATIONLAB_UPDATE") != "" { + updated, err := recordProvenance(doc, image, info.GitVersion, info.Platform, time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(updated), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + t.Logf("recorded %s (%s) in %s", info.GitVersion, image, path) + return + } + m := versionRow.FindStringSubmatch(doc) + if m == nil { + t.Fatalf("%s has no Server version row", path) + } + if m[1] != info.GitVersion { + t.Fatalf("the corpus was captured on %s and the lab cluster serves %s: "+ + "run `task lab-corpus-update` and review the corpus diff as that version's changes", m[1], info.GitVersion) + } +} + +// k3sServerVersion maps a k3s image to the version its API server reports. +func k3sServerVersion(image string) string { + tag := image[strings.LastIndex(image, ":")+1:] + return strings.Replace(tag, "-k3s", "+k3s", 1) +} + +// recordProvenance rewrites the three provenance rows of CLUSTER.md, and fails rather than append +// a row it cannot find, since the table's shape is what `task lab-bootstrap` reads the default +// k3s image from. +func recordProvenance(doc, image, version, platform string, now time.Time) (string, error) { + rows := []struct { + re *regexp.Regexp + row string + }{ + {imageRow, fmt.Sprintf("| k3d image | `%s` |", image)}, + {versionRow, fmt.Sprintf("| Server version | `%s` (%s) |", version, platform)}, + {capturedRow, fmt.Sprintf("| Captured at | %s |", now.Format(time.DateOnly))}, + } + for _, r := range rows { + if !r.re.MatchString(doc) { + return "", fmt.Errorf("CLUSTER.md has no row matching %s", r.re) + } + doc = r.re.ReplaceAllLiteralString(doc, r.row) + } + return doc, nil +} diff --git a/test/mutationlab/e2e/crd_conversion_test.go b/test/mutationlab/e2e/crd_conversion_test.go index f92f6abd5..f18ba1d34 100644 --- a/test/mutationlab/e2e/crd_conversion_test.go +++ b/test/mutationlab/e2e/crd_conversion_test.go @@ -6,9 +6,12 @@ package e2e import ( "context" + "encoding/base64" + "os" "testing" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -37,18 +40,20 @@ var ( Version: "v1", Resource: "customresourcedefinitions", } - vwcGVR = schema.GroupVersionResource{ - Group: "admissionregistration.k8s.io", - Version: "v1", - Resource: "validatingwebhookconfigurations", - } widgetV1 = schema.GroupVersionResource{Group: widgetGroup, Version: "v1", Resource: "widgets"} widgetV2 = schema.GroupVersionResource{Group: widgetGroup, Version: "v2", Resource: "widgets"} - vwcName = "gitops-reverser-validating-webhook" +) + +// The lab's Service, which serves /convert beside /validate-all under the same certificate +// (test/mutationlab/deploy/service.yaml). +const ( + labService = "mutation-capture-lab" + labNamespace = "mutation-capture-lab" + labPort = int64(443) ) // TestCRDConversion captures Row 14. It installs the two-version Widget CRD whose -// conversion webhook reuses the lab's admission cert/port at /convert, creates a +// conversion webhook calls the lab's /convert, creates a // Widget in v1, and captures the divergence: admission + audit see the submitted // v1 (spec.sizeBytes), the watch (opened on the v2 storage version) sees the // stored v2 (spec.size), and the conversion webhook is called in both directions. @@ -111,48 +116,46 @@ func TestCRDConversion(t *testing.T) { } // installWidgetCRD creates the two-version Widget CRD, pointing its conversion -// webhook at the lab by reusing the service and CA bundle the product's -// validating webhook already uses (so the lab needs no new certificate). It waits -// for the CRD to be Established and registers teardown. +// webhook at the lab's Service and trusting the lab CA from the stamp directory +// (LAB_CA_FILE). It waits for the CRD to be Established and registers a teardown +// that waits for the CRD to be gone, so the next install does not meet it still +// terminating. func (h *harness) installWidgetCRD(ctx context.Context, t *testing.T) { t.Helper() - svc, ca := h.admissionServiceAndCA(ctx, t) - crd := widgetCRDObject(svc, ca) + caFile := os.Getenv("LAB_CA_FILE") + if caFile == "" { + t.Fatal("LAB_CA_FILE not set; run via `task lab-capture`") + } + ca, err := os.ReadFile(caFile) + if err != nil { + t.Fatalf("read the lab CA: %v", err) + } + crd := widgetCRDObject(base64.StdEncoding.EncodeToString(ca)) if _, err := h.dyn.Resource(crdGVR).Create(ctx, crd, metav1.CreateOptions{}); err != nil { t.Fatalf("create Widget CRD: %v", err) } - t.Cleanup(func() { - _ = h.dyn.Resource(crdGVR).Delete(context.Background(), widgetCRD, metav1.DeleteOptions{}) - }) + t.Cleanup(func() { h.deleteWidgetCRD(t) }) h.waitForCRDEstablished(ctx, t) } -// admissionServiceAndCA reads the product validating webhook config and returns -// its clientConfig.service (name/namespace/port) and caBundle, so the Widget CRD -// conversion webhook can address the same lab endpoint over the same TLS trust. -func (h *harness) admissionServiceAndCA(ctx context.Context, t *testing.T) (map[string]any, string) { +func (h *harness) deleteWidgetCRD(t *testing.T) { t.Helper() - vwc, err := h.dyn.Resource(vwcGVR).Get(ctx, vwcName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("get validating webhook config %s: %v", vwcName, err) - } - webhooks, _, _ := unstructured.NestedSlice(vwc.Object, "webhooks") - if len(webhooks) == 0 { - t.Fatalf("validating webhook config %s has no webhooks", vwcName) - } - cc, _ := webhooks[0].(map[string]any)["clientConfig"].(map[string]any) - svc, _ := cc["service"].(map[string]any) - ca, _ := cc["caBundle"].(string) - if svc == nil || ca == "" { - t.Fatalf("validating webhook config %s missing service/caBundle", vwcName) + ctx := context.Background() + _ = h.dyn.Resource(crdGVR).Delete(ctx, widgetCRD, metav1.DeleteOptions{}) + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + if _, err := h.dyn.Resource(crdGVR).Get(ctx, widgetCRD, metav1.GetOptions{}); apierrors.IsNotFound(err) { + return + } + time.Sleep(time.Second) } - return svc, ca + t.Errorf("Widget CRD still present 60s after its deletion") } // widgetCRDObject builds the two-version Widget CRD. v1 carries spec.sizeBytes // (integer); v2 (storage) carries spec.size (string); the conversion webhook -// renames between them at the given service/path with the given CA bundle. -func widgetCRDObject(svc map[string]any, ca string) *unstructured.Unstructured { +// renames between them at the lab's /convert, trusting the given CA bundle. +func widgetCRDObject(ca string) *unstructured.Unstructured { intSpec := map[string]any{"type": "object", "properties": map[string]any{ "spec": map[string]any{"type": "object", "properties": map[string]any{ "sizeBytes": map[string]any{"type": "integer"}}}}} @@ -160,10 +163,10 @@ func widgetCRDObject(svc map[string]any, ca string) *unstructured.Unstructured { "spec": map[string]any{"type": "object", "properties": map[string]any{ "size": map[string]any{"type": "string"}}}}} convService := map[string]any{ - "name": svc["name"], - "namespace": svc["namespace"], + "name": labService, + "namespace": labNamespace, "path": "/convert", - "port": svc["port"], + "port": labPort, } return &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "apiextensions.k8s.io/v1", diff --git a/test/mutationlab/e2e/doc.go b/test/mutationlab/e2e/doc.go index 4a7e8ccc7..2111e0e24 100644 --- a/test/mutationlab/e2e/doc.go +++ b/test/mutationlab/e2e/doc.go @@ -7,5 +7,6 @@ // The driver is behind the `mutationlab_e2e` build tag so it never compiles into // the default `go test ./...` unit lane (which has no cluster). This file is the // untagged placeholder that keeps the package buildable when the tag is absent; -// `task lab-e2e` builds it with -tags mutationlab_e2e. +// `task lab-capture` builds it with -tags mutationlab_e2e against the lab's own +// cluster (test/mutationlab/Taskfile-lab.yml). package e2e diff --git a/test/mutationlab/e2e/harness_test.go b/test/mutationlab/e2e/harness_test.go index 7fb82f682..f0c5f88f2 100644 --- a/test/mutationlab/e2e/harness_test.go +++ b/test/mutationlab/e2e/harness_test.go @@ -54,7 +54,12 @@ func newHarness(t *testing.T) *harness { t.Helper() apiURL := os.Getenv("LAB_API_URL") if apiURL == "" { - t.Skip("LAB_API_URL not set; run via `task lab-e2e`") + t.Skip("LAB_API_URL not set; run via `task lab-capture`") + } + // The context is required, never defaulted: the current context may well be the e2e + // cluster, and the scenarios would then write there while the lab records nothing. + if os.Getenv("LAB_KUBECONTEXT") == "" { + t.Fatal("LAB_KUBECONTEXT not set; run via `task lab-capture`") } cfg, err := restConfig() if err != nil { @@ -256,10 +261,10 @@ func (h *harness) createNamespace(ctx context.Context, t *testing.T, ns string) } // ensureCaptureLive creates a labeled probe ConfigMap and waits until its -// admission record reaches the lab, confirming the webhook endpoint has finished -// cutting over to the lab pod after the image swap (the apiserver routes to the -// Service, whose endpoints lag a rollout). Without this, the first real scenario -// can silently miss its admission record. +// admission record reaches the lab, confirming the webhook Service routes to a +// ready lab pod (its endpoints lag a rollout). Without this, the first real +// scenario can silently miss its admission record. TestSourcesLive, which +// `task lab-bootstrap` runs, proves every other source. func (h *harness) ensureCaptureLive(t *testing.T) { t.Helper() ctx := context.Background() @@ -275,7 +280,7 @@ func (h *harness) ensureCaptureLive(t *testing.T) { return } } - t.Fatal("capture never went live: no admission record within 90s of the image swap") + t.Fatal("capture never went live: no admission record within 90s") } func hasSource(records []mutationlab.Record, src mutationlab.Source) bool { @@ -413,9 +418,8 @@ func (h *harness) probeWatch(t *testing.T, req watchProbeRequest) []mutationlab. } // do issues an HTTP request to the lab API, retrying transient connection errors. -// The API sits behind a kubectl port-forward that can drop and is restarted by a -// watchdog in the lab-e2e task, so a brief connection failure is expected, not -// fatal. +// The API sits behind a kubectl port-forward that can drop, so a brief connection +// failure is expected, not fatal. func (h *harness) do(method, path string) ([]byte, int, error) { return h.doJSON(method, path, nil) } diff --git a/test/mutationlab/e2e/sources_live_test.go b/test/mutationlab/e2e/sources_live_test.go new file mode 100644 index 000000000..2dc559350 --- /dev/null +++ b/test/mutationlab/e2e/sources_live_test.go @@ -0,0 +1,122 @@ +//go:build mutationlab_e2e + +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/ConfigButler/gitops-reverser/internal/mutationlab" +) + +// TestSourcesLive proves that every source the corpus records reaches the lab, each by a probe +// write it then finds among the records: the audit webhook backend, the admission webhook, the +// CRD conversion webhook, the aggregated API server, and the watch. `task lab-bootstrap` runs it +// alone, so a capture starts only on a cluster where all five deliver; the capture itself skips +// it. +// +// It exists because a source can go quiet without anything failing: the swapped-in lab of old +// lost its admission registration, and audit kept flowing, so only a check that names each source +// tells a missing one apart from a scenario that has none. +func TestSourcesLive(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + + t.Run("audit, admission, and watch", func(t *testing.T) { + s := h.newScenario(ctx, t, "live-configmap") + cm := &corev1.ConfigMap{ObjectMeta: s.meta("probe"), Data: map[string]string{"k": "v"}} + if _, err := h.kube.CoreV1().ConfigMaps(s.ns).Create(ctx, cm, metav1.CreateOptions{}); err != nil { + t.Fatalf("create probe ConfigMap: %v", err) + } + h.awaitSources(t, s.id, "", []sourceCheck{ + {"audit", func(rs []mutationlab.Record) bool { return firstSource(rs, mutationlab.SourceAudit) != nil }}, + {"admission", func(rs []mutationlab.Record) bool { + return firstSource(rs, mutationlab.SourceAdmission) != nil + }}, + {"watch", func(rs []mutationlab.Record) bool { return firstWatch(rs, "ADDED") != nil }}, + }) + }) + + t.Run("conversion", func(t *testing.T) { + h.installWidgetCRD(ctx, t) + s := h.newScenario(ctx, t, "live-conversion") + h.ensureWidgetWatchLive(ctx, t, s) + w := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": widgetGroup + "/v1", + "kind": "Widget", + "metadata": map[string]any{ + "name": "probe-2", "namespace": s.ns, "labels": map[string]any{scenarioLabel: s.id}, + }, + "spec": map[string]any{"sizeBytes": int64(1)}, + }} + if _, err := h.dyn.Resource(widgetV1).Namespace(s.ns).Create(ctx, w, metav1.CreateOptions{}); err != nil { + t.Fatalf("create probe Widget: %v", err) + } + h.awaitSources(t, s.id, "", []sourceCheck{ + {"conversion", func(rs []mutationlab.Record) bool { + return firstSource(rs, mutationlab.SourceConversion) != nil + }}, + }) + }) + + t.Run("aggregated API", func(t *testing.T) { + s := h.newScenario(ctx, t, "live-aggregated") + fl := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "wardle.example.com/v1alpha1", + "kind": "Flunder", + "metadata": map[string]any{ + "name": "probe", "namespace": s.ns, "labels": map[string]any{scenarioLabel: s.id}, + }, + "spec": map[string]any{"referenceType": "Flunder", "reference": "probe"}, + }} + if _, err := h.dyn.Resource(flunderGVR).Namespace(s.ns).Create(ctx, fl, metav1.CreateOptions{}); err != nil { + t.Fatalf("create probe flunder: %v", err) + } + // The namespace key as well: an aggregated API's audit event has no body to carry the + // scenario label, and its objectRef has no name for a create (row 15). + h.awaitSources(t, s.id, s.ns, []sourceCheck{ + {"aggregated API watch", func(rs []mutationlab.Record) bool { + return flunderRecordNamed(rs, mutationlab.SourceWatch, "ADDED", "probe") != nil + }}, + {"aggregated API audit", func(rs []mutationlab.Record) bool { + return flunderRecordNamed(rs, mutationlab.SourceAudit, "", "") != nil + }}, + }) + }) +} + +type sourceCheck struct { + name string + found func([]mutationlab.Record) bool +} + +// awaitSources polls the records of one scenario, unioned with those of namespace when it is set, +// until every check finds its record, and on timeout names the sources that never arrived. +func (h *harness) awaitSources(t *testing.T, id, namespace string, checks []sourceCheck) { + t.Helper() + const timeout = 90 * time.Second + deadline := time.Now().Add(timeout) + var missing []string + for time.Now().Before(deadline) { + records := h.fetchScenario(t, id, namespace) + missing = missing[:0] + for _, c := range checks { + if !c.found(records) { + missing = append(missing, c.name) + } + } + if len(missing) == 0 { + return + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("no %s record reached the lab within %s", strings.Join(missing, ", "), timeout) +} diff --git a/test/mutationlab/e2e/workload_scenarios_test.go b/test/mutationlab/e2e/workload_scenarios_test.go index 7010c613a..946ea69e3 100644 --- a/test/mutationlab/e2e/workload_scenarios_test.go +++ b/test/mutationlab/e2e/workload_scenarios_test.go @@ -22,9 +22,10 @@ import ( // Deployment for the /status (Row 5) and /scale (Row 6) subresources, and a Pod // for the two-event graceful delete (Row 7). // -// The headline finding here is mechanism *silence*. The lab reuses the product's -// audit policy and validating-webhook config verbatim (swap-image model), and -// those deliberately drop exactly these moments from the provenance mechanisms: +// The headline finding here is mechanism *silence*. The lab cluster uses the +// e2e audit policy and the validating-webhook match rules the corpus was captured +// under, and those deliberately drop exactly these moments from the provenance +// mechanisms: // // - the audit policy drops apps/*/status and core */status, so a /status write // is invisible to audit (Row 5); diff --git a/test/mutationlab/port-forward.sh b/test/mutationlab/port-forward.sh new file mode 100644 index 000000000..f9c535ebb --- /dev/null +++ b/test/mutationlab/port-forward.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Forward a host port to the mutation-capture lab's records API, detached with setsid so it +# outlives the task that starts it, and verified healthy before returning. +# +# A forward left from an earlier run is always replaced: it points at the pod that was current +# then, and a new image rolls a new pod. Only this cluster's forward is touched, so the e2e +# cluster's forwards keep running beside it. +# +# Inputs (env): +# - LAB_CTX (required): the lab's kube context +# - LAB_CS (required): its stamp directory, where the forward's log goes +# - LAB_NAMESPACE (optional, default mutation-capture-lab) +# - LAB_API_PORT (optional, default 18081): the host port +# - KUBECTL (optional): tool override + +: "${LAB_CTX:?LAB_CTX is required}" +: "${LAB_CS:?LAB_CS is required}" +LAB_NAMESPACE="${LAB_NAMESPACE:-mutation-capture-lab}" +LAB_API_PORT="${LAB_API_PORT:-18081}" +KUBECTL="${KUBECTL:-kubectl}" +REMOTE_PORT=8080 + +healthy() { curl -fsS "http://127.0.0.1:${LAB_API_PORT}/healthz" >/dev/null 2>&1; } + +pkill -f "port-forward --context ${LAB_CTX} .* ${LAB_API_PORT}:${REMOTE_PORT}" 2>/dev/null || true +sleep 1 + +"${KUBECTL}" --context "${LAB_CTX}" -n "${LAB_NAMESPACE}" rollout status deployment/mutation-capture-lab \ + --timeout=120s >/dev/null + +echo "🔌 Forwarding 127.0.0.1:${LAB_API_PORT} to the lab's records API…" +mkdir -p "${LAB_CS}" +pf_log="${LAB_CS}/port-forward.log" +# The context comes first so the pkill pattern above matches this cluster's forward only. +setsid "${KUBECTL}" port-forward --context "${LAB_CTX}" -n "${LAB_NAMESPACE}" --address 127.0.0.1 \ + deployment/mutation-capture-lab "${LAB_API_PORT}:${REMOTE_PORT}" >"${pf_log}" 2>&1 /dev/null; then + echo "❌ the lab port-forward exited:" >&2 + sed 's/^/ /' "${pf_log}" >&2 || true + exit 1 + fi + if healthy; then + echo "✅ The lab's records API answers on http://127.0.0.1:${LAB_API_PORT}" + exit 0 + fi + sleep 1 +done + +echo "❌ the lab port-forward was not healthy after 15s:" >&2 +sed 's/^/ /' "${pf_log}" >&2 || true +exit 1 diff --git a/test/mutationlab/selfcontained_test.go b/test/mutationlab/selfcontained_test.go new file mode 100644 index 000000000..585035801 --- /dev/null +++ b/test/mutationlab/selfcontained_test.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package mutationlab holds the mutation-capture lab's cluster wiring. This test keeps that +// wiring self-contained: the lab once borrowed the product's admission registration, certificate, +// and Deployment, and lost its admission source when the product removed them, with every check +// still green. +package mutationlab + +import ( + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "sigs.k8s.io/yaml" +) + +const repoRoot = "../.." + +// sharedPaths are the only files outside the lab's own directories that the lab's wiring may use. +// Each is shared on purpose, and changing one changes both clusters the same way. +var sharedPaths = []string{ + "test/e2e/cluster/start-cluster.sh", // creates both clusters + "test/e2e/cluster/audit/policy.yaml", // one audit policy, so the corpus shows what e2e audits + "test/e2e/setup/manifests/sample-apiserver", // the same aggregated API server + "test/mutationlab", // the lab's wiring + "cmd/mutation-capture-lab", "internal/mutationlab", // the lab binary +} + +// labFiles are the files the check reads: the lab binary and its wiring, without the corpus +// (data the API server wrote) and the Markdown (prose about the product is fine). +func labFiles(t *testing.T) []string { + t.Helper() + var files []string + for _, dir := range []string{"test/mutationlab", "cmd/mutation-capture-lab", "internal/mutationlab"} { + err := filepath.WalkDir(filepath.Join(repoRoot, dir), func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && d.Name() == "corpus" { + return filepath.SkipDir + } + if d.IsDir() || strings.HasSuffix(path, ".md") || strings.HasSuffix(path, "selfcontained_test.go") { + return nil + } + files = append(files, path) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + } + return files +} + +// TestLabTasksDependOnLabTasksOnly fails when a lab task runs or depends on a task the lab's +// Taskfile does not define, which is how the lab came to depend on the product's install. +func TestLabTasksDependOnLabTasksOnly(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(repoRoot, "test/mutationlab/Taskfile-lab.yml")) + if err != nil { + t.Fatal(err) + } + var tf struct { + Tasks map[string]struct { + Deps []any `json:"deps"` + Cmds []any `json:"cmds"` + } `json:"tasks"` + } + if err := yaml.Unmarshal(raw, &tf); err != nil { + t.Fatalf("parse Taskfile-lab.yml: %v", err) + } + for name, task := range tf.Tasks { + for _, ref := range append(taskRefs(task.Deps, true), taskRefs(task.Cmds, false)...) { + if _, ok := tf.Tasks[ref]; !ok { + t.Errorf("lab task %s calls %s, which Taskfile-lab.yml does not define", name, ref) + } + } + } +} + +// taskRefs returns the task names a deps or cmds list calls. A deps entry is a name or a +// {task: name} map; a cmds entry calls a task only in the map form. +func taskRefs(entries []any, bareNames bool) []string { + var refs []string + for _, e := range entries { + switch v := e.(type) { + case string: + if bareNames { + refs = append(refs, v) + } + case map[string]any: + if name, ok := v["task"].(string); ok { + refs = append(refs, name) + } + } + } + return refs +} + +var repoPath = regexp.MustCompile(`\b(?:test|hack|config|charts|cmd|internal|api)/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)*`) + +// TestLabUsesOnlySharedPaths fails when the lab's wiring (its Taskfile, manifests, and scripts) +// reads a repository path outside its own directories and the few it shares on purpose. +func TestLabUsesOnlySharedPaths(t *testing.T) { + for _, file := range labFiles(t) { + if strings.HasSuffix(file, ".go") { + // Go names packages, not files; the compiler and the name check below cover it. + continue + } + raw, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + for _, p := range uniq(repoPath.FindAllString(string(raw), -1)) { + if !shared(p) { + t.Errorf("%s uses %s, outside the lab and the paths it shares (sharedPaths)", rel(file), p) + } + } + } +} + +func shared(p string) bool { + for _, s := range sharedPaths { + if p == s || strings.HasPrefix(p, s+"/") { + return true + } + } + return false +} + +// TestLabNamesNoProductResource fails when the lab names something the product installs: its +// namespace, Deployment, Services, certificates, webhook registrations, or audit routes all carry +// the gitops-reverser name. Two spellings of it are the lab's own: the Go module path, and the lab +// cluster's name. +func TestLabNamesNoProductResource(t *testing.T) { + allowed := strings.NewReplacer( + "github.com/ConfigButler/gitops-reverser", "", + "gitops-reverser-capture-lab", "", + ) + for _, file := range labFiles(t) { + raw, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(raw), "\n") { + if strings.Contains(allowed.Replace(line), "gitops-reverser") { + t.Errorf("%s:%d names a product resource: %s", rel(file), i+1, strings.TrimSpace(line)) + } + } + } +} + +func uniq(in []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +func rel(path string) string { + r, err := filepath.Rel(repoRoot, path) + if err != nil { + return path + } + return r +} diff --git a/test/mutationlab/swap-image.sh b/test/mutationlab/swap-image.sh deleted file mode 100755 index 2f2e38880..000000000 --- a/test/mutationlab/swap-image.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Swap the deployed gitops-reverser controller to the mutation-capture lab image. -# -# The lab deliberately reuses the product's audit + admission webhook wiring (same -# URLs, same TLS cert mounts), so making the cluster capture with the lab instead -# of the controller is just this: import the lab image and patch the existing -# Deployment's image + entrypoint + args. No new audit policy, webhook config, or -# certificates are introduced — that is the whole point of the swap. -# -# Inputs (env): -# - CTX (required): kube context (k3d-) -# - CLUSTER_NAME (required): k3d cluster name (without the k3d- prefix) -# - NAMESPACE (required): controller namespace -# - LAB_IMAGE (required): lab image reference to import and run -# - K3D / KUBECTL (optional): tool overrides -# - CONTROLLER_DEPLOY_SELECTOR (optional): label selector for the controller Deployment -# - CONTROLLER_CONTAINER (optional): container name to patch (default: manager) - -: "${CTX:?CTX is required}" -: "${CLUSTER_NAME:?CLUSTER_NAME is required}" -: "${NAMESPACE:?NAMESPACE is required}" -: "${LAB_IMAGE:?LAB_IMAGE is required}" - -K3D="${K3D:-k3d}" -KUBECTL="${KUBECTL:-kubectl}" -SELECTOR="${CONTROLLER_DEPLOY_SELECTOR:-app.kubernetes.io/part-of=gitops-reverser}" -CONTAINER="${CONTROLLER_CONTAINER:-manager}" - -echo "Importing ${LAB_IMAGE} into k3d cluster ${CLUSTER_NAME}…" -"${K3D}" image import "${LAB_IMAGE}" -c "${CLUSTER_NAME}" - -deploy="$("${KUBECTL}" --context "${CTX}" -n "${NAMESPACE}" get deploy -l "${SELECTOR}" -o name | head -n1)" -[ -n "${deploy}" ] || { echo "no controller Deployment matching ${SELECTOR} in ${NAMESPACE}" >&2; exit 1; } -echo "Patching ${deploy} container '${CONTAINER}' to the lab image…" - -# Strategic-merge patch: container list is keyed by name, command/args replace -# wholesale. The cert mounts and the /readyz:8081 readiness probe are inherited -# from the product Deployment unchanged. -read -r -d '' patch < Date: Tue, 27 Oct 2026 11:56:23 +0000 Subject: [PATCH 2/3] test(mutationlab): recapture the corpus on the lab's own cluster, v1.36.4 Same k3s version as before (v1.36.4+k3s1); only the delivery path changed. 19 of 21 scenarios reproduce byte for byte. The audit events of the impersonated actors in rows 8a and 19 (15 files) each gain authenticationMetadata.impersonationConstraint: impersonate:user-info; the ServiceAccount-token patch in row 8a does not. The field comes from the API server, not the new wiring: a plain k3s v1.36.1 and v1.36.4 cluster with an audit log backend and none of the lab's wiring records it for the same impersonated write. Why the committed rows, captured on the e2e cluster, lacked it is not known. The fact-index tests that replay these rows pass unchanged. Co-Authored-By: Claude Opus 5.5 --- test/mutationlab/corpus/CLUSTER.md | 15 ++++++++++++--- .../deletion-intent-actor/audit.delete.yaml | 2 ++ .../audit.create.cm-exit-deleted.yaml | 2 ++ .../selector-membership/audit.create.cm-gone.yaml | 2 ++ .../selector-membership/audit.create.cm-held.yaml | 2 ++ .../audit.create.cm-member.yaml | 2 ++ .../audit.create.cm-outsider.yaml | 2 ++ .../audit.delete.cm-exit-deleted.yaml | 2 ++ .../selector-membership/audit.delete.cm-gone.yaml | 2 ++ .../selector-membership/audit.delete.cm-held.yaml | 2 ++ .../selector-membership/audit.patch.1.yaml | 2 ++ .../selector-membership/audit.patch.3.yaml | 2 ++ .../selector-membership/audit.patch.4.yaml | 2 ++ .../selector-membership/audit.update.1.yaml | 2 ++ .../selector-membership/audit.update.2.yaml | 2 ++ .../selector-membership/audit.update.3.yaml | 2 ++ 16 files changed, 42 insertions(+), 3 deletions(-) diff --git a/test/mutationlab/corpus/CLUSTER.md b/test/mutationlab/corpus/CLUSTER.md index 5a528f983..674f63e02 100644 --- a/test/mutationlab/corpus/CLUSTER.md +++ b/test/mutationlab/corpus/CLUSTER.md @@ -7,15 +7,24 @@ reviewing the diff *is* the changelog of "what changed in Kubernetes between these versions" (see [the design](../../../docs/spec/mutation-capture-lab-design.md#validating-new-kubernetes-versions)). -The lab reuses the main e2e cluster (`task lab-e2e` swaps the controller image), -so this provenance tracks the k3s image used for the committed capture. +The lab runs on its own cluster, created on the k3s image below unless +`LAB_K3S_IMAGE` names another. `task lab-corpus-update` writes the three rows of +the table; `task lab-capture` fails when the cluster serves another version than +the one recorded here. | Field | Value | |---|---| | k3d image | `rancher/k3s:v1.36.4-k3s1` | | Server version | `v1.36.4+k3s1` (linux/amd64) | -| Captured at | 2026-09-28 | +| Captured at | 2026-10-27 | Moving from `v1.36.1+k3s1` (captured 2026-06-25) to `v1.36.4+k3s1` changed one committed shape: the garbage collector's `userAgent` in `configmap/owner-ref-cascade/`. Every other scenario reproduced unchanged. + +Recapturing on the lab's own cluster (2026-10-27, same version) changed one +field: each audit event of an impersonated actor in rows 8a and 19 +(`configmap/deletion-intent-actor/`, `configmap/selector-membership/`) gained +`authenticationMetadata.impersonationConstraint: impersonate:user-info`. A plain +k3s v1.36.1 and v1.36.4 cluster, with an audit log backend instead of the lab, +records it for the same request. Every other scenario reproduced unchanged. diff --git a/test/mutationlab/corpus/configmap/deletion-intent-actor/audit.delete.yaml b/test/mutationlab/corpus/configmap/deletion-intent-actor/audit.delete.yaml index 227ddec45..8e50b0961 100644 --- a/test/mutationlab/corpus/configmap/deletion-intent-actor/audit.delete.yaml +++ b/test/mutationlab/corpus/configmap/deletion-intent-actor/audit.delete.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-human" to User "alice@example.com"' validate-all.configbutler.ai/scenario: deletion-intent-actor auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-exit-deleted.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-exit-deleted.yaml index 9569aa032..41b006e17 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-exit-deleted.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-exit-deleted.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-gone.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-gone.yaml index 070bc4a8d..3d9273bd7 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-gone.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-gone.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-held.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-held.yaml index d0fcdedf1..1293ee9f2 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-held.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-held.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-member.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-member.yaml index c4c34842b..07e70f066 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-member.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-member.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-outsider.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-outsider.yaml index 5136ea131..a9bfcd32f 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-outsider.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-outsider.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-exit-deleted.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-exit-deleted.yaml index 072e16b92..7f4c37b90 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-exit-deleted.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-exit-deleted.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-car" to User "carol@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-gone.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-gone.yaml index 8187607d4..e219a7c84 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-gone.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-gone.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-bob" to User "bob@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-held.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-held.yaml index a32843a74..06b82fce7 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-held.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-held.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.1.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.1.yaml index ed9f67e0e..7057f86f7 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.1.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.1.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-bob" to User "bob@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.3.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.3.yaml index ac540d0f0..31c4b9945 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.3.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.3.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.4.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.4.yaml index a9e3591b4..7a7d77e33 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.4.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.4.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-bob" to User "bob@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.update.1.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.update.1.yaml index f3969a5d0..6c427dbdd 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.update.1.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.update.1.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.update.2.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.update.2.yaml index ad5006865..881561c41 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.update.2.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.update.2.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-bob" to User "bob@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.update.3.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.update.3.yaml index 73fb87fd6..2afe88ad8 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.update.3.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.update.3.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated From c7707d51824ac1046a7dcd17f15454a83a45a6db Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Tue, 27 Oct 2026 12:04:36 +0000 Subject: [PATCH 3/3] test(mutationlab): capture the corpus on k3s v1.37.1, the e2e cluster's version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the Kubernetes version changed since the previous commit, so this diff is the 1.36.4 → 1.37.1 changelog for what the corpus records. Each change reproduced in a second capture: - deletecollection: the response's list resourceVersion is the last deleted object's own, where 1.36 returned a newer one, so the watch DELETED placeholders shift by one; - graceful pod delete: PodReadyToStartContainers gains reason PodSandboxNotReady and a message; - owner-ref cascade: the garbage collector's userAgent names v1.37.1. The fact-index tests that replay the corpus pass unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/spec/mutation-capture-lab-design.md | 2 +- test/mutationlab/corpus/CLUSTER.md | 17 +++++++++++++++-- .../audit.deletecollection.yaml | 2 +- .../deletecollection/watch.deleted.cm-a.yaml | 2 +- .../deletecollection/watch.deleted.cm-b.yaml | 2 +- .../deletecollection/watch.deleted.cm-c.yaml | 2 +- .../audit.delete.cm-child.yaml | 2 +- .../pod/graceful-delete/watch.deleted.yaml | 4 ++++ 8 files changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/spec/mutation-capture-lab-design.md b/docs/spec/mutation-capture-lab-design.md index 9aa9d9c64..f28b399e7 100644 --- a/docs/spec/mutation-capture-lab-design.md +++ b/docs/spec/mutation-capture-lab-design.md @@ -8,7 +8,7 @@ webhooks** — at every interesting moment — and commits those structures as a **corpus**. It is not a second implementation of GitOps Reverser; it is reference material and a regression harness. -The corpus is captured against **k8s v1.36.4+k3s1** (recorded in +The corpus is captured against **k8s v1.37.1+k3s1** (recorded in `test/mutationlab/corpus/CLUSTER.md`). **All seventeen** catalogued scenarios are captured today (see the [Difficult Cases Catalog](#difficult-cases-catalog)), plus the rows that extend it, which `test/mutationlab/README.md` indexes. diff --git a/test/mutationlab/corpus/CLUSTER.md b/test/mutationlab/corpus/CLUSTER.md index 674f63e02..66a26d27e 100644 --- a/test/mutationlab/corpus/CLUSTER.md +++ b/test/mutationlab/corpus/CLUSTER.md @@ -14,8 +14,8 @@ the one recorded here. | Field | Value | |---|---| -| k3d image | `rancher/k3s:v1.36.4-k3s1` | -| Server version | `v1.36.4+k3s1` (linux/amd64) | +| k3d image | `rancher/k3s:v1.37.1-k3s1` | +| Server version | `v1.37.1+k3s1` (linux/amd64) | | Captured at | 2026-10-27 | Moving from `v1.36.1+k3s1` (captured 2026-06-25) to `v1.36.4+k3s1` changed one @@ -28,3 +28,16 @@ field: each audit event of an impersonated actor in rows 8a and 19 `authenticationMetadata.impersonationConstraint: impersonate:user-info`. A plain k3s v1.36.1 and v1.36.4 cluster, with an audit log backend instead of the lab, records it for the same request. Every other scenario reproduced unchanged. + +Moving from `v1.36.4+k3s1` to `v1.37.1+k3s1` (2026-10-27, the e2e cluster's +version) changed three shapes, each reproduced by a second capture: + +- `configmap/deletecollection/`: the `deletecollection` response's list + resourceVersion is now the last deleted object's own resourceVersion, where + 1.36 returned a newer one, so the watch DELETED placeholders shift by one; +- `pod/graceful-delete/`: the deleted pod's `PodReadyToStartContainers` + condition now carries `reason: PodSandboxNotReady` and a message; +- `configmap/owner-ref-cascade/`: the garbage collector's `userAgent` names the + new version. + +Every other scenario reproduced unchanged. diff --git a/test/mutationlab/corpus/configmap/deletecollection/audit.deletecollection.yaml b/test/mutationlab/corpus/configmap/deletecollection/audit.deletecollection.yaml index a5c1b50e0..64c07d382 100644 --- a/test/mutationlab/corpus/configmap/deletecollection/audit.deletecollection.yaml +++ b/test/mutationlab/corpus/configmap/deletecollection/audit.deletecollection.yaml @@ -48,7 +48,7 @@ responseObject: uid: kind: ConfigMapList metadata: - resourceVersion: + resourceVersion: responseStatus: code: 200 metadata: {} diff --git a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-a.yaml b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-a.yaml index fde7f8cf3..70f66e10f 100644 --- a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-a.yaml +++ b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-a.yaml @@ -23,6 +23,6 @@ object: time: name: cm-a namespace: - resourceVersion: + resourceVersion: uid: type: DELETED diff --git a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-b.yaml b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-b.yaml index f09f9277b..4ff490827 100644 --- a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-b.yaml +++ b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-b.yaml @@ -23,6 +23,6 @@ object: time: name: cm-b namespace: - resourceVersion: + resourceVersion: uid: type: DELETED diff --git a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-c.yaml b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-c.yaml index 0f7d5d31c..1b4533da3 100644 --- a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-c.yaml +++ b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-c.yaml @@ -23,6 +23,6 @@ object: time: name: cm-c namespace: - resourceVersion: + resourceVersion: uid: type: DELETED diff --git a/test/mutationlab/corpus/configmap/owner-ref-cascade/audit.delete.cm-child.yaml b/test/mutationlab/corpus/configmap/owner-ref-cascade/audit.delete.cm-child.yaml index f4c7b3c75..b75e1be44 100644 --- a/test/mutationlab/corpus/configmap/owner-ref-cascade/audit.delete.cm-child.yaml +++ b/test/mutationlab/corpus/configmap/owner-ref-cascade/audit.delete.cm-child.yaml @@ -51,5 +51,5 @@ user: - system:authenticated uid: username: system:serviceaccount:kube-system:generic-garbage-collector -userAgent: k3s/v1.36.4+k3s1 (linux/amd64) kubernetes/4dedb15/system:serviceaccount:kube-system:generic-garbage-collector +userAgent: k3s/v1.37.1+k3s1 (linux/amd64) kubernetes/356c025/system:serviceaccount:kube-system:generic-garbage-collector verb: delete diff --git a/test/mutationlab/corpus/pod/graceful-delete/watch.deleted.yaml b/test/mutationlab/corpus/pod/graceful-delete/watch.deleted.yaml index 3e588443d..9f1e4b333 100644 --- a/test/mutationlab/corpus/pod/graceful-delete/watch.deleted.yaml +++ b/test/mutationlab/corpus/pod/graceful-delete/watch.deleted.yaml @@ -61,7 +61,9 @@ object: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} + f:message: {} f:observedGeneration: {} + f:reason: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: @@ -131,7 +133,9 @@ object: conditions: - lastProbeTime: null lastTransitionTime: + message: pod sandbox is not ready observedGeneration: 2 + reason: PodSandboxNotReady status: "False" type: PodReadyToStartContainers - lastProbeTime: null