diff --git a/.github/workflows/mutation-lab.yml b/.github/workflows/mutation-lab.yml new file mode 100644 index 000000000..b24284bf4 --- /dev/null +++ b/.github/workflows/mutation-lab.yml @@ -0,0 +1,78 @@ +# Mutation-capture lab: creates the lab's own k3d cluster, runs every scenario, and fails on any +# difference from the committed corpus under test/mutationlab/corpus/ +# (docs/design/mutation-capture-lab-own-cluster.md). +# +# Nightly and on demand rather than per PR. Its job is to notice when something stops the lab +# from working, or makes it record something else, before the next Kubernetes upgrade depends on +# it; a break shows up within a day of the change that caused it. Nothing here publishes, and the +# run needs no secrets beyond a read token to pull the CI container. +name: Mutation lab + +on: + schedule: + - cron: "17 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: mutation-lab-${{ github.ref }} + cancel-in-progress: true + +env: + # The CI base container main last published (ci.yml moves :latest only from main): the same + # Go, kubectl, k3d, and task the e2e legs run with. + CI_CONTAINER: ghcr.io/configbutler/gitops-reverser-ci:latest + CI_WORKDIR: /workspaces/${{ github.event.repository.name }} + +jobs: + capture: + name: Capture and compare to the corpus + runs-on: ubuntu-latest + timeout-minutes: 40 + permissions: + contents: read + packages: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Login to Docker registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Run lab-capture in the CI container + # One container for the whole run: the lab's kube context and its records-API forward + # live only as long as it does. Host networking reaches the k3d API port and the forward. + run: | + docker run --rm \ + --network host \ + -v "${GITHUB_WORKSPACE}:${{ env.CI_WORKDIR }}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -w "${{ env.CI_WORKDIR }}" \ + -e HOST_PROJECT_PATH=${{ github.workspace }} \ + ${{ env.CI_CONTAINER }} \ + bash -c " + git config --global --add safe.directory ${{ env.CI_WORKDIR }} + task lab-capture + " + + - name: Show the lab on failure + if: failure() + run: | + docker run --rm \ + --network host \ + -v "${GITHUB_WORKSPACE}:${{ env.CI_WORKDIR }}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -w "${{ env.CI_WORKDIR }}" \ + ${{ env.CI_CONTAINER }} \ + bash -c " + k3d kubeconfig merge gitops-reverser-capture-lab --kubeconfig-switch-context=false \ + --kubeconfig-merge-default >/dev/null 2>&1 || exit 0 + kubectl --context k3d-gitops-reverser-capture-lab get pods -A -o wide || true + kubectl --context k3d-gitops-reverser-capture-lab -n mutation-capture-lab \ + logs deployment/mutation-capture-lab --tail=200 || true + " diff --git a/Taskfile.yml b/Taskfile.yml index 02ac12fc1..f8c8c7c2f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -24,6 +24,10 @@ includes: taskfile: ./test/e2e/Taskfile-e2e.yml dir: . flatten: true + lab: + taskfile: ./test/mutationlab/Taskfile-lab.yml + dir: . + flatten: true tasks: default: diff --git a/cmd/mutation-capture-lab/main.go b/cmd/mutation-capture-lab/main.go index cf18fcbb0..cb0d94249 100644 --- a/cmd/mutation-capture-lab/main.go +++ b/cmd/mutation-capture-lab/main.go @@ -6,9 +6,11 @@ // deliberately NOT a second GitOps Reverser; see // docs/spec/mutation-capture-lab-design.md. // -// It deliberately serves the SAME webhook URLs as the product -// (/validate-all, /audit-webhook) so a lab deployment can swap the product image -// without reconfiguring the cluster's admission/audit wiring. +// It runs on a cluster of its own (test/mutationlab/Taskfile-lab.yml), which +// delivers to it the way any cluster delivers: the API server's audit webhook +// backend posts to /audit-webhook over mutual TLS, a ValidatingWebhookConfiguration +// sends admission reviews to /validate-all, and the scenario CRD's conversion +// webhook calls /convert. package main import ( @@ -121,12 +123,6 @@ func buildServers( auditMux := http.NewServeMux() auditMux.Handle("/audit-webhook", audit) - // Audit routes are NAMED in the e2e cluster: its bootstrap kubeconfig posts to - // /audit-webhook/ (the chart's local KubeProvider), because the bare - // path is the product's shared annotation-routed endpoint. The lab records what - // arrives regardless of which source it was addressed to, so it takes the whole - // subtree — an exact-path-only mux 404s every event the cluster sends. - auditMux.Handle("/audit-webhook/", audit) return []server{ {name: "admission", certDir: cfg.admissionCert, srv: &http.Server{ diff --git a/docs/ci-overview.md b/docs/ci-overview.md index e25464e26..3f5547e4c 100644 --- a/docs/ci-overview.md +++ b/docs/ci-overview.md @@ -20,6 +20,7 @@ so local and CI results cannot drift apart. | Release & publish | [release.yml](../.github/workflows/release.yml) tail jobs | after trusted validation is green | `GITHUB_TOKEN` + OIDC | packages, releases, attestations | Version (release-please), **retag** the CI-built multi-arch digests to semver + `latest` (zero rebuilds), publish chart, sign, attest | | Hygiene | [scorecard.yml](../.github/workflows/scorecard.yml) | weekly + `main` | none | security-events | OpenSSF Scorecard supply-chain checks | | Hygiene | [pr-title.yml](../.github/workflows/pr-title.yml) | `pull_request` (including `edited`) | none | none | The PR title is a Conventional Commit, since it becomes the squash-merge subject release-please classifies | +| Hygiene | [mutation-lab.yml](../.github/workflows/mutation-lab.yml) | nightly + manual | `GITHUB_TOKEN` (read) | none | The mutation-capture lab on its own cluster, compared to its committed corpus ([design](design/mutation-capture-lab-own-cluster.md)) | Three properties are worth calling out: @@ -164,5 +165,5 @@ GitHub's OIDC identity and logged in the public Rekor transparency log. | Tool versions | [.devcontainer/Dockerfile](../.devcontainer/Dockerfile) (single source for devcontainer *and* CI) | | Validation pipeline | [.github/workflows/ci.yml](../.github/workflows/ci.yml) | | Release pipeline | [.github/workflows/release.yml](../.github/workflows/release.yml), [.github/workflows/release-pr.yml](../.github/workflows/release-pr.yml) | -| Hygiene | [.github/workflows/scorecard.yml](../.github/workflows/scorecard.yml), [.github/dependabot.yml](../.github/dependabot.yml) | +| Hygiene | [.github/workflows/scorecard.yml](../.github/workflows/scorecard.yml), [.github/workflows/mutation-lab.yml](../.github/workflows/mutation-lab.yml), [.github/dependabot.yml](../.github/dependabot.yml) | | Release process details | [.github/RELEASES.md](../.github/RELEASES.md) | diff --git a/docs/design/mutation-capture-lab-own-cluster.md b/docs/design/mutation-capture-lab-own-cluster.md index bcbeb4f0e..83776658b 100644 --- a/docs/design/mutation-capture-lab-own-cluster.md +++ b/docs/design/mutation-capture-lab-own-cluster.md @@ -1,6 +1,6 @@ # Mutation-capture lab on its own cluster -Status: proposed, 2026-10-27. Not built. +Status: built, 2026-10-27. [As built](#as-built) lists where the build departs from this design. The [mutation-capture lab](../spec/mutation-capture-lab-design.md) records what Kubernetes emits through watches, audit, admission, CRD conversion, and an aggregated API, and commits it as a @@ -15,7 +15,7 @@ the requirements that keep the capture faithful and the lab working. ## What broke, and why it could The lab never owned its wiring. `task lab-e2e` swapped the lab image into the product's controller -Deployment on the e2e cluster ([`swap-image.sh`](../../test/mutationlab/swap-image.sh)) and borrowed +Deployment on the e2e cluster (`swap-image.sh`, since removed) and borrowed everything around it: | The lab needs | It came from | Since `fd6f6933` | @@ -181,7 +181,7 @@ context rather than the current one. Otherwise bootstrapping the lab would move | the lab binary, recorders, normalizer, corpus, and driver | unchanged, apart from the client-CA flag, the context, and the readiness check | | the content-hashed lab image and its build | unchanged | | `lab-port-forward.sh` | with the lab's namespace and selector | -| the audit kubeconfig template | from `generate-audit-webhook-kubeconfig.sh`, fed from files instead of Secrets | +| the audit kubeconfig | written by the certificate command, not by `generate-audit-webhook-kubeconfig.sh` (see [As built](#as-built)) | Not reused: `inject-webhook-tls.sh`, whose node restart the lab avoids, and the Flux stack. @@ -230,10 +230,28 @@ Each one names the failure it prevents. 7. The scheduled CI job, the static self-containment check, and the lab's README and [spec](../spec/mutation-capture-lab-design.md#isolated-test-setup) rewritten to match. -## Open questions - -- **CI cadence.** Nightly plus manual dispatch is the proposal. A per-PR job would catch a break - in the PR that causes it, for a few runner minutes per PR. -- **Memory beside e2e.** One k3s node with the lab and the sample API server should stay near - 1 GiB, but that is an estimate until measured. If it is more, `lab-capture` can take the e2e lock - as well, so the two run in turn. +## As built + +- **Two Services, not one.** `mutation-capture-lab` is a ClusterIP Service for admission, + conversion (443 to 9443), and the records API (8080); `mutation-capture-lab-audit` is the NodePort + 30444 to 9444. A NodePort Service gives every port it lists a node port. +- **The certificate command writes the audit kubeconfig.** `generate-audit-webhook-kubeconfig.sh` + defaults to the product's names and reads Secrets; requirement 5 keeps the lab off it. +- **The webhook keeps the name `validate-all.configbutler.ai`.** The API server keys the admission + recorder's audit annotation by it (`validate-all.configbutler.ai/scenario`) and quotes it in row + 12's rejection, so the name is part of the corpus, like the match rules of requirement 3. +- **The checks are tests in the driver.** `TestSourcesLive` is the five-source check: + `lab-bootstrap` runs it, and `lab-capture` skips it. `TestClusterProvenance` writes `CLUSTER.md` + on an update and fails a capture when the cluster serves another version. The static check is + [`selfcontained_test.go`](../../test/mutationlab/selfcontained_test.go), in `task test`. +- **The capture on v1.36.4 differs in one field.** 19 of 21 scenarios reproduced the corpus byte for + byte. The other two, rows 8a and 19, gained `authenticationMetadata.impersonationConstraint: + impersonate:user-info` on each of their 15 impersonated audit events, and on nothing else. A plain + k3s v1.36.1 and v1.36.4 cluster with an audit log backend and none of the lab's wiring records the + same field for the same request, so it comes from the API server (constrained impersonation, beta + in 1.36), not from the new delivery path. Why the committed rows, captured on the e2e cluster, + lack it is not known. +- **Memory.** The lab cluster's node used about 600 MiB with the lab and the sample API server + running, so `lab-capture` does not take the e2e lock. +- **CI.** [`mutation-lab.yml`](../../.github/workflows/mutation-lab.yml) runs `lab-capture` nightly + and on manual dispatch. diff --git a/docs/proud.md b/docs/proud.md index c49fc1bef..6edbf8dad 100644 --- a/docs/proud.md +++ b/docs/proud.md @@ -96,7 +96,8 @@ be **SSH-signed**. It even captures resources served by an **aggregated API serv - A **self-ratcheting coverage gate** (`cover-check`) that only ever moves the floor up, with merged unit + e2e coverage reported to Codecov. - **Three install modes** — config-dir, Helm, and plain manifests — all exercised by e2e. -- A **mutation-capture lab** (`lab-e2e`) for stress-testing capture correctness. +- A **mutation-capture lab** (`lab-capture`), on a cluster of its own, that records what Kubernetes + emits and diffs it against a committed corpus. - `actionlint` on the CI workflows, Conventional Commits driving `release-please`. - A genuinely deep [docs/architecture.md](architecture.md) that answers the "but how does it actually…" questions instead of hand-waving. diff --git a/docs/spec/mutation-capture-lab-design.md b/docs/spec/mutation-capture-lab-design.md index 05d92bcf1..f28b399e7 100644 --- a/docs/spec/mutation-capture-lab-design.md +++ b/docs/spec/mutation-capture-lab-design.md @@ -8,14 +8,15 @@ webhooks** — at every interesting moment — and commits those structures as a **corpus**. It is not a second implementation of GitOps Reverser; it is reference material and a regression harness. -The corpus is captured against **k8s v1.36.4+k3s1** (recorded in +The corpus is captured against **k8s v1.37.1+k3s1** (recorded in `test/mutationlab/corpus/CLUSTER.md`). **All seventeen** catalogued scenarios are captured today (see the [Difficult Cases Catalog](#difficult-cases-catalog)), plus the rows that extend it, which `test/mutationlab/README.md` indexes. -The lab reuses the product's webhook URLs (`/validate-all`, `/audit-webhook`) on the same ports and -TLS mounts, so capturing is just an **image swap** on an already-prepared e2e cluster — no new audit -policy, webhook config, or certificates. The binary lives under `cmd/mutation-capture-lab/`, the +The lab runs on a k3d cluster of its own, which delivers audit, admission, conversion, and the +aggregated API to it through their real mechanisms +([design](../design/mutation-capture-lab-own-cluster.md)). The binary lives under +`cmd/mutation-capture-lab/`, the recorders/normalizer/store/ record model under `internal/mutationlab/`, and the live-cluster driver plus golden corpus under `test/mutationlab/` (driver behind the `mutationlab_e2e` build tag, so the normal lane never needs a @@ -386,13 +387,14 @@ sometimes itself the behavior under test. ### Golden workflow -- **Default run:** capture → normalize → compare against the committed file; a mismatch fails the test - and prints a unified diff. -- **Update run:** `MUTATIONLAB_UPDATE=1 task lab-e2e` (or `task lab-corpus-update`) rewrites the corpus - from the live capture. -- **Validating a new Kubernetes version:** bump the k3d image, run the update, and review the `corpus/` - diff — it *is* the behavioral changelog for the upgrade, scoped to exactly the behaviors GitOps - Reverser depends on, with volatile fields normalized away. An empty diff is a positive result. +- **Default run:** `task lab-capture`: capture → normalize → compare against the committed file; a + mismatch fails the test and prints a unified diff. +- **Update run:** `task lab-corpus-update` rewrites the corpus from the live capture, and the k3s + image, server version, and date in `CLUSTER.md`. +- **Validating a new Kubernetes version:** recreate the lab cluster with + `task lab-bootstrap LAB_K3S_IMAGE=`, run the update, and review the `corpus/` diff — it + *is* the behavioral changelog for the upgrade, scoped to exactly the behaviors GitOps Reverser + depends on, with volatile fields normalized away. An empty diff is a positive result. ## Record model @@ -438,13 +440,17 @@ not) after the request — never guessed from the payload alone. The lab gets its own everything and reuses only the integration *shape* of the real install: - **Separate binary** (`cmd/mutation-capture-lab/main.go`), built as its own image. -- **Swap the image, reuse the wiring.** Rather than authoring its own audit policy, webhook configs, - and certificates, the lab serves the **same** webhook URLs as the product on the same ports and TLS - mounts, so `task lab-e2e` integrates by swapping the controller image on the already-prepared e2e - cluster (`test/mutationlab/swap-image.sh`). The cluster is left running the lab image; restore with - `task clean-cluster && task test-e2e`. -- **Separate, opt-in, serial task targets** (`task lab-e2e`, `task lab-corpus-update`) — not invoked by - `task test-e2e` or the default CI lane; the driver is behind the `mutationlab_e2e` build tag. +- **Its own cluster.** The lab runs on a k3d cluster of its own, `gitops-reverser-capture-lab`, + with its own certificates, `ValidatingWebhookConfiguration`, Service, and RBAC + (`test/mutationlab/deploy/`), and nothing of the product. The API server's audit webhook backend, + the admission registration, the Widget CRD's conversion webhook, and the `wardle` aggregated API + deliver to it as they would on any cluster; it shares only the cluster script, the audit policy, + and the sample API server's manifests with e2e. See + [the mutation-capture lab on its own cluster](../design/mutation-capture-lab-own-cluster.md). +- **Separate, opt-in, serial tasks** (`task lab-bootstrap`, `task lab-capture`, + `task lab-corpus-update`) in `test/mutationlab/Taskfile-lab.yml`, not invoked by `task test-e2e` + or the per-PR CI lanes; a scheduled workflow runs `lab-capture`. The driver is behind the + `mutationlab_e2e` build tag. Because audit delivery is asynchronous and batched, a late event from one scenario can land after the next has started. The lab **isolates** scenarios rather than trusting a clean slate: @@ -502,7 +508,7 @@ so each row remains a deliberate stop-and-read point. Write Git commits · reuse the GitOps Reverser controller runtime · implement the full `WatchRule`/`GitTarget` model · solve HA/retention/multi-cluster · prove every resource behaves identically · build a **mutating** admission recorder (cited, not re-proven) · re-prove **Cite**-half -contracts · hand-author corpus files · wire the lab into `task test-e2e` or the default CI lane · add +contracts · hand-author corpus files · wire the lab into `task test-e2e` or the per-PR CI lanes · add lab-only behavior to the Helm chart. ## Related diff --git a/docs/tasks-overview.md b/docs/tasks-overview.md index 36d042180..02c23058f 100644 --- a/docs/tasks-overview.md +++ b/docs/tasks-overview.md @@ -311,9 +311,13 @@ flowchart LR FULL --> ALLURE["allure-e2e-results/report/open"] FULL --> COV["e2e-coverage-collect
(when E2E_COVERAGE=1)"] - PREP --> LAB["lab-e2e / lab-corpus-update
(serial opt-in)"] ``` +The mutation-capture lab is not in this graph: it runs on a cluster of its own, through +`task lab-bootstrap`, `lab-capture`, and `lab-corpus-update` +([`test/mutationlab/Taskfile-lab.yml`](../test/mutationlab/Taskfile-lab.yml)), and no task here +depends on it or it on them. + ## Why the DAG pays off: only what changed re-runs Because each task declares its real inputs and outputs, Task skips any step already up to diff --git a/hack/e2e/lab-port-forward.sh b/hack/e2e/lab-port-forward.sh deleted file mode 100755 index 7fc427f90..000000000 --- a/hack/e2e/lab-port-forward.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Establish a robust port-forward to the mutation-capture lab's records API -# (the controller pod's :8081), mirroring hack/e2e/setup-port-forwards.sh: the -# forward is detached with setsid so it survives this shell, verified for both -# liveness and a healthy /healthz, and reused on the fast path when already up. -# -# The lab API is not on the controller Service (which exposes only audit/webhook/ -# metrics), so this forwards to the Deployment, which re-resolves to the current -# pod — important because each image swap rolls a new pod. -# -# Inputs (env): -# - CTX / E2E_KUBECONTEXT (required): kube context -# - NAMESPACE (required): controller namespace -# - LAB_API_PORT (optional, default 18081): local port -# - CONTROLLER_DEPLOY_SELECTOR (optional): label selector for the controller -# - REMOTE_PORT (optional, default 8081): the lab API port in the pod - -KUBE_CONTEXT="${E2E_KUBECONTEXT:-${CTX:-}}" -NAMESPACE="${NAMESPACE:?NAMESPACE is required}" -LAB_API_PORT="${LAB_API_PORT:-18081}" -SELECTOR="${CONTROLLER_DEPLOY_SELECTOR:-app.kubernetes.io/part-of=gitops-reverser}" -REMOTE_PORT="${REMOTE_PORT:-8081}" - -if [[ -z "${KUBE_CONTEXT}" ]]; then - KUBE_CONTEXT="$(kubectl config current-context 2>/dev/null || true)" -fi -[[ -n "${KUBE_CONTEXT}" ]] || { echo "❌ kube context is required (set CTX/E2E_KUBECONTEXT)"; exit 1; } - -healthy() { curl -fsS "http://127.0.0.1:${LAB_API_PORT}/healthz" >/dev/null 2>&1; } - -# No fast-path reuse: the lab pod is rolled by every image swap, so a surviving -# forward from a previous run points at the now-terminated old pod. Always tear -# down any existing forward and re-establish against the current ready pod. -echo "🧹 Cleaning up any stale lab port-forward on ${LAB_API_PORT}…" -pkill -f "kubectl.*port-forward.*${LAB_API_PORT}:${REMOTE_PORT}" 2>/dev/null || true -sleep 1 - -# The Deployment carries SELECTOR, but its Pods do not — derive the pod selector -# from the Deployment's own .spec.selector.matchLabels. -deploy="$(kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" get deploy -l "${SELECTOR}" -o name | head -n1)" -[[ -n "${deploy}" ]] || { echo "❌ no controller Deployment matching ${SELECTOR}"; exit 1; } -pod_selector="$(kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" get "${deploy}" \ - -o go-template='{{range $k,$v := .spec.selector.matchLabels}}{{$k}}={{$v}},{{end}}')" -pod_selector="${pod_selector%,}" -[[ -n "${pod_selector}" ]] || { echo "❌ ${deploy} has no pod selector"; exit 1; } - -echo "⏳ Waiting for the lab pod to be ready (selector ${pod_selector})…" -deadline=$((SECONDS + 120)) -pod="" -while (( SECONDS < deadline )); do - pod="$(kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" get pods -l "${pod_selector}" \ - -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' \ - | head -n1)" - if [[ -n "${pod}" ]] && kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" \ - wait --for=condition=ready "pod/${pod}" --timeout=15s >/dev/null 2>&1; then - break - fi - pod="" - sleep 2 -done -[[ -n "${pod}" ]] || { echo "❌ no ready lab pod matching ${pod_selector}"; exit 1; } - -echo "🔌 Starting lab API port-forward (${LAB_API_PORT} -> ${REMOTE_PORT})…" -pf_log="/tmp/lab-api-pf.log" -setsid kubectl --context "${KUBE_CONTEXT}" -n "${NAMESPACE}" port-forward --address 127.0.0.1 \ - "pod/${pod}" "${LAB_API_PORT}:${REMOTE_PORT}" >"${pf_log}" 2>&1 < /dev/null & -pf_pid=$! - -for i in {1..15}; do - if ! kill -0 "${pf_pid}" 2>/dev/null; then - echo "❌ lab port-forward process died:"; sed 's/^/ /' "${pf_log}" 2>/dev/null || true - exit 1 - fi - if healthy; then - echo "✅ lab API port-forward verified on http://127.0.0.1:${LAB_API_PORT}" - exit 0 - fi - sleep 1 -done - -echo "❌ lab API port-forward did not become healthy after 15s:"; sed 's/^/ /' "${pf_log}" 2>/dev/null || true -exit 1 diff --git a/internal/mutationlab/labserver/api.go b/internal/mutationlab/labserver/api.go index bb5d75a45..9111feb34 100644 --- a/internal/mutationlab/labserver/api.go +++ b/internal/mutationlab/labserver/api.go @@ -47,8 +47,7 @@ func (a *API) Routes() *http.ServeMux { mux.HandleFunc("/records", a.handleRecords) mux.HandleFunc("/watch-probe", a.handleWatchProbe) mux.HandleFunc("/healthz", a.handleHealthz) - // /readyz mirrors /healthz so a lab deployment can satisfy the product - // deployment's readiness probe unchanged when it swaps in the lab image. + // /readyz mirrors /healthz; the lab Deployment's readiness probe reads it. mux.HandleFunc("/readyz", a.handleHealthz) return mux } diff --git a/test/e2e/Taskfile-e2e.yml b/test/e2e/Taskfile-e2e.yml index f30e5a509..aeed265ed 100644 --- a/test/e2e/Taskfile-e2e.yml +++ b/test/e2e/Taskfile-e2e.yml @@ -31,14 +31,6 @@ vars: CONTAINER_TOOL: '{{.CONTAINER_TOOL | default "docker"}}' TASK_BIN: '{{.TASK_BIN | default "task"}}' E2E_LOCAL_IMAGE: '{{.E2E_LOCAL_IMAGE | default "gitops-reverser:e2e-local"}}' - # Mutation-capture lab (opt-in, serial; see the lab-* tasks at the end of file). - # The image tag is a content hash of the lab sources, so each change produces a - # unique tag. The swap patch then changes the deployment's image ref and rolls - # out naturally; an unchanged source reuses the same tag (no needless rollout). - LAB_IMAGE_HASH: - sh: cat $(find cmd/mutation-capture-lab internal/mutationlab -type f -name '*.go' | sort) go.mod go.sum test/mutationlab/Dockerfile 2>/dev/null | sha256sum | cut -c1-12 - LAB_IMAGE: '{{.LAB_IMAGE | default (printf "mutation-capture-lab:e2e-%s" .LAB_IMAGE_HASH)}}' - LAB_API_PORT: '{{.LAB_API_PORT | default "18081"}}' PROJECT_IMAGE: sh: | if [ -n "${PROJECT_IMAGE:-}" ]; then @@ -1571,66 +1563,3 @@ tasks: kubectl --context "{{.CTX}}" apply -k "{{.DEMO_MANIFESTS_DIR}}" touch "{{.CS}}/demo.ready" - # ---------------------------------------------------------------------------- - # Mutation-capture lab (docs/spec/mutation-capture-lab-design.md). - # - # Opt-in and SERIAL. These targets are NOT invoked by `task test-e2e` and are - # NOT part of the default CI lane. The lab deliberately reuses the product's - # audit + admission webhook wiring (same URLs, same TLS cert mounts), so it just - # swaps the controller image for the lab image on the already-prepared cluster — - # no new audit policy, webhook config, or certificates. After running the lab, - # restore the product with `task clean-cluster` then `task test-e2e`. - # ---------------------------------------------------------------------------- - lab-build-image: - desc: Build the mutation-capture lab image (separate from the product image). - cmds: - - '{{.CONTAINER_TOOL}} build -f test/mutationlab/Dockerfile -t {{.LAB_IMAGE}} .' - - _lab-deployed: - # Reuse the product bring-up + install (same cluster, same audit + admission - # wiring), then swap in the lab image. Depends on _controller-deployed so the - # cluster, CRDs, certs, audit policy, and webhook configs are all in place. - deps: - - _controller-deployed - - lab-build-image - cmds: - - | - CTX="{{.CTX}}" CLUSTER_NAME="{{.CLUSTER_NAME}}" NAMESPACE="{{.NAMESPACE}}" \ - LAB_IMAGE="{{.LAB_IMAGE}}" K3D="{{.K3D}}" KUBECTL="{{.KUBECTL}}" \ - CONTROLLER_DEPLOY_SELECTOR="{{.CONTROLLER_DEPLOY_SELECTOR}}" \ - CONTROLLER_CONTAINER="{{.CONTROLLER_CONTAINER}}" \ - bash test/mutationlab/swap-image.sh - - lab-e2e: - desc: Run the mutation-capture lab end-to-end (serial). Swaps the lab image onto the cluster. - deps: - - _lab-deployed - cmds: - - task: _lab-run-driver - - lab-corpus-update: - desc: Regenerate the lab corpus from a live capture (serial, MUTATIONLAB_UPDATE=1). - deps: - - _lab-deployed - cmds: - - task: _lab-run-driver - vars: - MUTATIONLAB_UPDATE: '1' - - _lab-run-driver: - vars: - MUTATIONLAB_UPDATE: '{{.MUTATIONLAB_UPDATE | default ""}}' - cmds: - - | - set -euo pipefail - # Robust, detached port-forward to the lab API, reusing the established - # e2e port-forward pattern (setsid + ready-pod wait + health verify). The - # driver additionally retries transient connection errors. - CTX="{{.CTX}}" NAMESPACE="{{.NAMESPACE}}" LAB_API_PORT="{{.LAB_API_PORT}}" \ - CONTROLLER_DEPLOY_SELECTOR="{{.CONTROLLER_DEPLOY_SELECTOR}}" \ - bash hack/e2e/lab-port-forward.sh - # Serial by construction: a single `go test` package run, no Ginkgo --procs. - LAB_API_URL="http://127.0.0.1:{{.LAB_API_PORT}}" \ - LAB_KUBECONTEXT="{{.CTX}}" \ - MUTATIONLAB_UPDATE="{{.MUTATIONLAB_UPDATE}}" \ - go test -tags mutationlab_e2e -count=1 -timeout 10m -v ./test/mutationlab/e2e/ diff --git a/test/e2e/cluster/README.md b/test/e2e/cluster/README.md index 6f7360c73..68e205bf0 100644 --- a/test/e2e/cluster/README.md +++ b/test/e2e/cluster/README.md @@ -31,6 +31,10 @@ audit webhook setup restarts the server container to load the final webhook conf keeps that restart from taking the manager, Valkey and Gitea down with it. `K3D_AGENT_COUNT=0` still builds a single-node cluster, with no taint. +`K3D_SWITCH_CONTEXT=false` creates the cluster without making it the kubeconfig's current context; +the mutation-capture lab's own cluster ([`test/mutationlab/Taskfile-lab.yml`](../../mutationlab/Taskfile-lab.yml)) +is created that way, so bootstrapping it never moves `kubectl`, or an e2e run, onto it. + It also disables these packaged k3s components by default: - `traefik` diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh index e62ddb410..ff4d18d8b 100644 --- a/test/e2e/cluster/start-cluster.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -16,6 +16,9 @@ KUBE_APISERVER_MAX_MUTATING_REQUESTS_INFLIGHT="${KUBE_APISERVER_MAX_MUTATING_REQ # accepted for a single-node cluster, which leaves the server untainted. K3D_AGENT_COUNT="${K3D_AGENT_COUNT:-1}" K3S_IMAGE="${K3S_IMAGE:-rancher/k3s:v1.37.1-k3s1}" +# false leaves the kubeconfig's current context where it was, so a second cluster (the +# mutation-capture lab) can be created without moving kubectl, or an e2e run, onto it. +K3D_SWITCH_CONTEXT="${K3D_SWITCH_CONTEXT:-true}" AUDIT_DIR_REL="${AUDIT_DIR_REL:-test/e2e/cluster/audit}" K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" REPO_PWD="$(pwd -P)" @@ -265,7 +268,7 @@ create_cluster() { --servers 1 --agents "${K3D_AGENT_COUNT}" --kubeconfig-update-default - --kubeconfig-switch-context + "--kubeconfig-switch-context=${K3D_SWITCH_CONTEXT}" -v "${audit_host_dir}:/etc/kubernetes/audit@server:0" ) if [ -n "${K3D_API_PORT}" ]; then @@ -280,17 +283,20 @@ create_cluster() { } merge_kubeconfig() { - # Ensure kubeconfig is present and current-context points at this cluster + # Ensure kubeconfig holds this cluster's context, and is current unless K3D_SWITCH_CONTEXT=false k3d kubeconfig merge "${CLUSTER_NAME}" \ - --kubeconfig-switch-context \ + "--kubeconfig-switch-context=${K3D_SWITCH_CONTEXT}" \ --kubeconfig-merge-default >/dev/null } rewrite_kubeconfig_for_devcontainer() { - local cluster_entry server host port + local context_name cluster_entry server host port - cluster_entry="$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')" - server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" + # The named context, not the current one: with K3D_SWITCH_CONTEXT=false the current + # context is some other cluster, whose endpoint this must not touch. + context_name="$(cluster_context_name)" + cluster_entry="$(kubectl config view --context "${context_name}" --minify -o jsonpath='{.clusters[0].name}')" + server="$(kubectl config view --context "${context_name}" --minify -o jsonpath='{.clusters[0].cluster.server}')" # Parse https://HOST:PORT if [[ "$server" =~ ^https://([^:/]+):([0-9]+)$ ]]; then diff --git a/test/e2e/setup/manifests/kustomization.yaml b/test/e2e/setup/manifests/kustomization.yaml index b2d215899..4e1757983 100644 --- a/test/e2e/setup/manifests/kustomization.yaml +++ b/test/e2e/setup/manifests/kustomization.yaml @@ -5,12 +5,4 @@ resources: - prometheus/instance.yaml - prometheus/prometheus-rbac.yaml - prometheus/servicemonitor.yaml - - sample-apiserver/namespace.yaml - - sample-apiserver/sa.yaml - - sample-apiserver/deployment.yaml - - sample-apiserver/service.yaml - - sample-apiserver/apiservice.yaml - - sample-apiserver/auth-delegator.yaml - - sample-apiserver/auth-reader.yaml - - sample-apiserver/rbac.yaml - - sample-apiserver/rbac-bind.yaml + - sample-apiserver diff --git a/test/e2e/setup/manifests/sample-apiserver/kustomization.yaml b/test/e2e/setup/manifests/sample-apiserver/kustomization.yaml new file mode 100644 index 000000000..2b5fdcc7a --- /dev/null +++ b/test/e2e/setup/manifests/sample-apiserver/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The wardle sample aggregated API server. The e2e setup and the mutation-capture lab +# (test/mutationlab/Taskfile-lab.yml) both apply this directory, so the two clusters +# serve the same aggregated API. +resources: + - namespace.yaml + - sa.yaml + - deployment.yaml + - service.yaml + - apiservice.yaml + - auth-delegator.yaml + - auth-reader.yaml + - rbac.yaml + - rbac-bind.yaml diff --git a/test/mutationlab/Dockerfile b/test/mutationlab/Dockerfile index 12e0315b0..859fee1c5 100644 --- a/test/mutationlab/Dockerfile +++ b/test/mutationlab/Dockerfile @@ -1,5 +1,5 @@ -# Mutation-capture lab image. Built and deployed only by `task lab-e2e`; it is -# never part of the product image, the Helm chart, or the default CI lane. +# Mutation-capture lab image. Built by `task lab-build-image` and run on the lab's +# own cluster; it is never part of the product image or the Helm chart. FROM golang:1.27.3 AS builder ARG TARGETOS diff --git a/test/mutationlab/README.md b/test/mutationlab/README.md index 73c3c6bdd..e42feddcc 100644 --- a/test/mutationlab/README.md +++ b/test/mutationlab/README.md @@ -12,7 +12,10 @@ GitOps Reverser — see [the design](../../docs/spec/mutation-capture-lab-design | `cmd/mutation-capture-lab/` | the lab binary (recorders + records API) | | `internal/mutationlab/` | record model, normalizer, store, corpus, recorders, API | | `test/mutationlab/Dockerfile` | the lab image (built only by `task lab-build-image`) | -| `test/mutationlab/swap-image.sh` | swaps the lab image onto the controller Deployment | +| `test/mutationlab/Taskfile-lab.yml` | the lab's tasks: its own cluster, deployment, and capture | +| `test/mutationlab/deploy/` | the lab's namespace, RBAC, Deployment, Services, and webhook registration | +| `test/mutationlab/tools/lab-certs/` | issues the lab's CA, serving certificate, and the API server's audit client certificate | +| `test/mutationlab/deploy.sh`, `port-forward.sh` | deploy onto the lab cluster; forward its records API | | `test/mutationlab/e2e/` | the live-cluster driver (build tag `mutationlab_e2e`) | | `test/mutationlab/corpus/` | the committed golden corpus + `CLUSTER.md` provenance | @@ -69,29 +72,38 @@ transport-only events can be scenario-attributed — see the [watch-first ingestion architecture](../../docs/finished/watch-first-ingestion-architecture.md) design notes. -> ⚠️ **The "no audit" rows record this cluster's audit POLICY, not the API server.** The lab runs -> against the already-prepared e2e cluster and reuses its policy +> ⚠️ **The "no audit" rows record this cluster's audit POLICY, not the API server.** The lab cluster +> uses the e2e audit policy > ([`test/e2e/cluster/audit/policy.yaml`](../e2e/cluster/audit/policy.yaml)), which drops `pods` and > every `*/status` at `level: None` as runtime noise. So rows 5 and 7 show what a cluster configured > like this one does not tell us — a `DELETE` on a pod is an audited request like any other when the > policy asks for it. Read them as "excluded here", not as "unknowable". Re-capturing either row > against a policy that includes those types would settle it by measurement. -## How it integrates: swap the image, reuse the wiring +## How it integrates: a cluster of its own -The lab serves the **same** webhook URLs as the product — -`/validate-all` and `/audit-webhook` — on the same ports and TLS cert mounts. So -making a cluster capture with the lab is just swapping the controller image: no -new audit policy, webhook config, or certificates. `task lab-e2e` does this on the -already-prepared e2e cluster, then drives the scenarios serially. +The lab runs on its own k3d cluster, `gitops-reverser-capture-lab`, which runs the lab, the +`wardle` sample aggregated API server, and nothing of the product +([design](../../docs/design/mutation-capture-lab-own-cluster.md)). Every source arrives the way it +does on any cluster: -Reusing the wiring means reusing whatever the cluster's audit kubeconfig points at, and -this one names its route: it posts to `/audit-webhook/gitops-reverser.local`, the route of the -`local` KubeProvider in `gitops-reverser`, because the bare path is the product's shared annotation-routed -endpoint. The lab therefore serves the whole `/audit-webhook/` subtree and records what -arrives whichever source it was addressed to. Serving only the exact bare path 404s every -event the cluster sends — which reads as a lab that captures watches and admission but -never a single audit record, in every scenario at once. +- the API server's **audit webhook backend** posts to `https://127.0.0.1:30444/audit-webhook`, a + NodePort on its own node, over mutual TLS: it presents a client certificate from the lab CA, and + the lab refuses a connection without one; +- the `ValidatingWebhookConfiguration` `mutation-capture-lab` sends **admission** reviews to the + lab's Service; +- the scenario Widget CRD's **conversion** webhook calls the same Service at `/convert`; +- the lab **watches** through its own ServiceAccount, with read-only RBAC on what it watches. + +`lab-certs` issues the certificates before the cluster exists, so the audit kubeconfig the API +server reads at startup is already the final one and the cluster never restarts. The cluster is +created by the same [`start-cluster.sh`](../e2e/cluster/start-cluster.sh) as the e2e cluster, with +the same audit flags and policy, on one node, and without switching the current kube context. + +Nothing in the lab's wiring names a product resource or depends on a product task; +[`selfcontained_test.go`](selfcontained_test.go) enforces that in `task test`. The lab used to swap +its image into the product's Deployment and borrow its webhook registration and certificates, and +lost its admission source the day the product removed them, with every check still green. Row 15 (aggregated API write) is what settled the body-enrichment question: the official audit event for an aggregated-API write carries an empty body, yet the @@ -101,28 +113,35 @@ retired — so the lab no longer serves or records that endpoint. ## Running it (opt-in, serial) -These targets are **not** part of `task test-e2e` or the default CI lane. +These tasks are not part of `task test-e2e` or the per-PR CI lanes; the +[mutation lab workflow](../../.github/workflows/mutation-lab.yml) runs `lab-capture` nightly. ```bash -# Prepare the e2e cluster + product wiring, swap in the lab image, capture, diff: -task lab-e2e - -# Accept a new/changed capture as the golden corpus: -task lab-corpus-update +task lab-bootstrap # certificates, cluster, image, deployment, and a live check of all five sources +task lab-capture # run the scenarios and compare the capture to the corpus +task lab-corpus-update # accept a new or changed capture as the corpus, CLUSTER.md included +task lab-status # the cluster, its k3s version, the lab pod, and how many records it holds +task lab-cleanup # delete the cluster and its stamps ``` -`task lab-e2e` leaves the cluster running the lab image. To restore the product: - -```bash -task clean-cluster && task test-e2e -``` +The lab cluster can run beside the e2e cluster: its own name, stamps +(`.stamps/cluster/k3d-gitops-reverser-capture-lab/`), run lock, and records-API port (18081). It +needs no restore afterwards; the e2e cluster is never touched. The unit tests for the lab packages run in the normal lane (`task test`); the -`test/mutationlab/e2e/` driver is behind the `mutationlab_e2e` build tag and only -runs under `task lab-e2e`. +`test/mutationlab/e2e/` driver is behind the `mutationlab_e2e` build tag and runs only through the +lab's tasks. ## Validating a new Kubernetes version -Bump the k3d image, run `task lab-corpus-update`, and review the `corpus/` diff: -it is the behavioral changelog for the upgrade, scoped to exactly the fine-grained -behaviors GitOps Reverser depends on. An empty diff is a positive result. +```bash +task lab-cleanup +task lab-bootstrap LAB_K3S_IMAGE=rancher/k3s:v1.38.0-k3s1 +task lab-corpus-update +git diff test/mutationlab/corpus/ +``` + +The `corpus/` diff is the behavioral changelog for the upgrade, scoped to exactly the fine-grained +behaviors GitOps Reverser depends on. An empty diff is a positive result. Without `LAB_K3S_IMAGE`, a +new lab cluster runs the k3s image `CLUSTER.md` records, and `lab-capture` fails when the cluster +serves another version than the corpus was captured on. diff --git a/test/mutationlab/Taskfile-lab.yml b/test/mutationlab/Taskfile-lab.yml new file mode 100644 index 000000000..1672dc747 --- /dev/null +++ b/test/mutationlab/Taskfile-lab.yml @@ -0,0 +1,211 @@ +# yaml-language-server: $schema=https://taskfile.dev/schema.json +version: '3' + +# The mutation-capture lab on its own k3d cluster +# (docs/design/mutation-capture-lab-own-cluster.md). It runs the lab and nothing of the product, +# beside the e2e cluster if one is up: its own cluster name, stamps, lock, and port, and it never +# switches the current kube context. +# +# Self-contained by rule: no task here depends on a task outside this file, and nothing here names +# a product resource. test/mutationlab/selfcontained_test.go enforces both in `task test`. +# +# task lab-bootstrap certificates, cluster, image, deployment, and a live check of every source +# task lab-capture run the scenarios and compare the capture to the corpus +# task lab-corpus-update run the scenarios and write the capture as the corpus, CLUSTER.md included +# task lab-status the cluster, its k3s version, the lab pod, and how many records it holds +# task lab-cleanup delete the cluster and its stamp directory +# +# Testing a new Kubernetes version: +# task lab-cleanup && task lab-bootstrap LAB_K3S_IMAGE=rancher/k3s: && task lab-corpus-update + +vars: + LAB_CLUSTER_NAME: '{{.LAB_CLUSTER_NAME | default "gitops-reverser-capture-lab"}}' + LAB_CTX: 'k3d-{{.LAB_CLUSTER_NAME}}' + LAB_CS: '.stamps/cluster/{{.LAB_CTX}}' + LAB_NAMESPACE: mutation-capture-lab + LAB_API_PORT: '{{.LAB_API_PORT | default "18081"}}' + LAB_CONTAINER_TOOL: '{{.LAB_CONTAINER_TOOL | default "docker"}}' + LAB_AUDIT_POLICY_SOURCE: test/e2e/cluster/audit/policy.yaml + # The k3s image a new lab cluster is created with: LAB_K3S_IMAGE when given, else the one the + # committed corpus was captured on, as test/mutationlab/corpus/CLUSTER.md records it. An + # existing cluster keeps the image it was created with, recorded in its stamps. + LAB_K3S_IMAGE_RECORDED: + sh: sed -n 's/^| k3d image | `\([^`]*\)` |$/\1/p' test/mutationlab/corpus/CLUSTER.md + LAB_K3S_IMAGE_REQUESTED: '{{.LAB_K3S_IMAGE | default ""}}' + # The image tag is a content hash of the lab's sources, so a change rolls the lab pod and an + # unchanged source keeps it running. + LAB_IMAGE_HASH: + sh: >- + cat $(find cmd/mutation-capture-lab internal/mutationlab -type f -name '*.go' | sort) + go.mod go.sum test/mutationlab/Dockerfile | sha256sum | cut -c1-12 + LAB_IMAGE: 'mutation-capture-lab:{{.LAB_IMAGE_HASH}}' + +tasks: + lab-bootstrap: + desc: Create the mutation-capture lab's own cluster, deploy the lab, and prove every source is live. + cmds: + - task: _lab-deployed + - task: _lab-port-forward + # The probe writes of every source, found among the records: audit, admission, conversion, + # the aggregated API, and the watch. + - task: _lab-driver + vars: + LAB_RUN: '^TestSourcesLive$' + + lab-capture: + desc: Run the lab scenarios on the lab cluster and compare the capture to the committed corpus. + cmds: + - task: lab-bootstrap + - task: _lab-driver + vars: + LAB_SKIP: '^TestSourcesLive$' + + lab-corpus-update: + desc: Run the lab scenarios and write the capture as the corpus, CLUSTER.md included. + cmds: + - task: lab-bootstrap + - task: _lab-driver + vars: + LAB_SKIP: '^TestSourcesLive$' + MUTATIONLAB_UPDATE: '1' + + lab-status: + desc: Show the lab cluster, its k3s version, the lab pod, and how many records it holds. + cmds: + - | + if ! k3d cluster list "{{.LAB_CLUSTER_NAME}}" >/dev/null 2>&1; then + echo "No lab cluster '{{.LAB_CLUSTER_NAME}}'. Create it with: task lab-bootstrap" + exit 0 + fi + k3d cluster list "{{.LAB_CLUSTER_NAME}}" + echo + echo "k3s image: $(cat "{{.LAB_CS}}/k3s-image" 2>/dev/null || echo unknown)" + kubectl --context "{{.LAB_CTX}}" version 2>/dev/null | grep -i '^server' || echo "Server: unreachable" + echo + kubectl --context "{{.LAB_CTX}}" -n "{{.LAB_NAMESPACE}}" get pods -o wide || true + echo + if records="$(curl -fsS "http://127.0.0.1:{{.LAB_API_PORT}}/records" 2>/dev/null)"; then + count="$(printf '%s' "${records}" | grep -o '"source":"[a-z]*"' | wc -l)" + echo "Records held: ${count}" + else + echo "Records held: unknown (no forward on {{.LAB_API_PORT}}; task lab-bootstrap starts one)" + fi + + lab-cleanup: + desc: Delete the lab cluster and its stamp directory. + cmds: + - pkill -f "port-forward --context {{.LAB_CTX}} " 2>/dev/null || true + # Unconditional: an interrupted create leaves state `k3d cluster list` does not show, and + # delete exits 0 when there is nothing to delete. + - k3d cluster delete "{{.LAB_CLUSTER_NAME}}" + - rm -rf "{{.LAB_CS}}" + + lab-build-image: + desc: Build the mutation-capture lab image. + status: + - '{{.LAB_CONTAINER_TOOL}} image inspect "{{.LAB_IMAGE}}" >/dev/null 2>&1' + cmds: + - '{{.LAB_CONTAINER_TOOL}} build -f test/mutationlab/Dockerfile -t "{{.LAB_IMAGE}}" .' + + _lab-certs: + # Issued before the cluster exists, so the audit kubeconfig the API server reads at startup is + # already the final one. + status: + - test -f "{{.LAB_CS}}/certs/ca.crt" + - test -f "{{.LAB_CS}}/certs/tls.crt" + - test -f "{{.LAB_CS}}/certs/tls.key" + - test -f "{{.LAB_CS}}/audit/webhook-config.yaml" + cmds: + - >- + go run ./test/mutationlab/tools/lab-certs + --cert-dir "{{.LAB_CS}}/certs" + --audit-kubeconfig "{{.LAB_CS}}/audit/webhook-config.yaml" + --namespace "{{.LAB_NAMESPACE}}" + + _lab-cluster-ready: + deps: + - _lab-certs + preconditions: + # A stamped cluster that does not answer is wedged (host suspend, OOM, a killed run); + # recreating it under the same stamps would only hide that. + - sh: | + test -f "{{.LAB_CS}}/ready" || exit 0 + kubectl --context "{{.LAB_CTX}}" --request-timeout=5s get nodes \ + -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null \ + | awk 'BEGIN{n=0;t=0} {t++; if ($0=="True") n++} END{exit !(t>0 && n==t)}' + msg: | + The lab cluster '{{.LAB_CLUSTER_NAME}}' is stamped as ready but its nodes are not Ready. + Recreate it: task lab-cleanup && task lab-bootstrap + - sh: | + test -f "{{.LAB_CS}}/ready" || exit 0 + test -z "{{.LAB_K3S_IMAGE_REQUESTED}}" || test "$(cat "{{.LAB_CS}}/k3s-image")" = "{{.LAB_K3S_IMAGE_REQUESTED}}" + msg: | + The lab cluster runs another k3s image than LAB_K3S_IMAGE={{.LAB_K3S_IMAGE_REQUESTED}}. + Recreate it on that image: task lab-cleanup && task lab-bootstrap LAB_K3S_IMAGE={{.LAB_K3S_IMAGE_REQUESTED}} + status: + - test -f "{{.LAB_CS}}/ready" + cmds: + - mkdir -p "{{.LAB_CS}}/audit" + # The one tracked audit policy, shared with e2e, so the corpus shows what the product's + # clusters audit. + - cp "{{.LAB_AUDIT_POLICY_SOURCE}}" "{{.LAB_CS}}/audit/policy.yaml" + - | + image="{{.LAB_K3S_IMAGE_REQUESTED | default .LAB_K3S_IMAGE_RECORDED}}" + test -n "${image}" || { echo "no k3s image: set LAB_K3S_IMAGE or record one in CLUSTER.md" >&2; exit 1; } + # One node: the lab never restarts its server, so it needs no agent to keep pods away + # from a restart. + CLUSTER_NAME="{{.LAB_CLUSTER_NAME}}" \ + K3S_IMAGE="${image}" \ + AUDIT_DIR_REL="{{.LAB_CS}}/audit" \ + K3D_AGENT_COUNT=0 \ + K3D_SWITCH_CONTEXT=false \ + bash test/e2e/cluster/start-cluster.sh + printf '%s\n' "${image}" > "{{.LAB_CS}}/k3s-image" + touch "{{.LAB_CS}}/ready" + + _lab-deployed: + deps: + - _lab-cluster-ready + - lab-build-image + cmds: + - | + LAB_CLUSTER_NAME="{{.LAB_CLUSTER_NAME}}" LAB_CTX="{{.LAB_CTX}}" LAB_CS="{{.LAB_CS}}" \ + LAB_IMAGE="{{.LAB_IMAGE}}" LAB_NAMESPACE="{{.LAB_NAMESPACE}}" \ + CONTAINER_TOOL="{{.LAB_CONTAINER_TOOL}}" \ + bash test/mutationlab/deploy.sh + + _lab-port-forward: + run: always + cmds: + - | + LAB_CTX="{{.LAB_CTX}}" LAB_CS="{{.LAB_CS}}" LAB_NAMESPACE="{{.LAB_NAMESPACE}}" \ + LAB_API_PORT="{{.LAB_API_PORT}}" \ + bash test/mutationlab/port-forward.sh + + _lab-driver: + # always: the root Taskfile runs each task once per invocation, and this one runs twice in a + # capture, once for the live check and once for the scenarios. + run: always + vars: + LAB_RUN: '{{.LAB_RUN | default ""}}' + LAB_SKIP: '{{.LAB_SKIP | default ""}}' + MUTATIONLAB_UPDATE: '{{.MUTATIONLAB_UPDATE | default ""}}' + cmds: + - | + mkdir -p "{{.LAB_CS}}" + # Serial by construction (one `go test` package run), and one run per cluster at a time: + # scenarios clear the shared record store. + rc=0 + LAB_API_URL="http://127.0.0.1:{{.LAB_API_PORT}}" \ + LAB_KUBECONTEXT="{{.LAB_CTX}}" \ + LAB_CA_FILE="$(pwd)/{{.LAB_CS}}/certs/ca.crt" \ + LAB_K3S_IMAGE="$(cat "{{.LAB_CS}}/k3s-image")" \ + MUTATIONLAB_UPDATE="{{.MUTATIONLAB_UPDATE}}" \ + flock --nonblock --conflict-exit-code 75 "{{.LAB_CS}}/lab.lock" \ + go test -tags mutationlab_e2e -count=1 -timeout 15m -v \ + {{if .LAB_RUN}}-run '{{.LAB_RUN}}'{{end}} {{if .LAB_SKIP}}-skip '{{.LAB_SKIP}}'{{end}} \ + ./test/mutationlab/e2e/ || rc=$? + if [ "${rc}" -eq 75 ]; then + echo "Another lab run holds {{.LAB_CS}}/lab.lock; wait for it to finish." >&2 + fi + exit "${rc}" diff --git a/test/mutationlab/corpus/CLUSTER.md b/test/mutationlab/corpus/CLUSTER.md index 5a528f983..66a26d27e 100644 --- a/test/mutationlab/corpus/CLUSTER.md +++ b/test/mutationlab/corpus/CLUSTER.md @@ -7,15 +7,37 @@ reviewing the diff *is* the changelog of "what changed in Kubernetes between these versions" (see [the design](../../../docs/spec/mutation-capture-lab-design.md#validating-new-kubernetes-versions)). -The lab reuses the main e2e cluster (`task lab-e2e` swaps the controller image), -so this provenance tracks the k3s image used for the committed capture. +The lab runs on its own cluster, created on the k3s image below unless +`LAB_K3S_IMAGE` names another. `task lab-corpus-update` writes the three rows of +the table; `task lab-capture` fails when the cluster serves another version than +the one recorded here. | Field | Value | |---|---| -| k3d image | `rancher/k3s:v1.36.4-k3s1` | -| Server version | `v1.36.4+k3s1` (linux/amd64) | -| Captured at | 2026-09-28 | +| k3d image | `rancher/k3s:v1.37.1-k3s1` | +| Server version | `v1.37.1+k3s1` (linux/amd64) | +| Captured at | 2026-10-27 | Moving from `v1.36.1+k3s1` (captured 2026-06-25) to `v1.36.4+k3s1` changed one committed shape: the garbage collector's `userAgent` in `configmap/owner-ref-cascade/`. Every other scenario reproduced unchanged. + +Recapturing on the lab's own cluster (2026-10-27, same version) changed one +field: each audit event of an impersonated actor in rows 8a and 19 +(`configmap/deletion-intent-actor/`, `configmap/selector-membership/`) gained +`authenticationMetadata.impersonationConstraint: impersonate:user-info`. A plain +k3s v1.36.1 and v1.36.4 cluster, with an audit log backend instead of the lab, +records it for the same request. Every other scenario reproduced unchanged. + +Moving from `v1.36.4+k3s1` to `v1.37.1+k3s1` (2026-10-27, the e2e cluster's +version) changed three shapes, each reproduced by a second capture: + +- `configmap/deletecollection/`: the `deletecollection` response's list + resourceVersion is now the last deleted object's own resourceVersion, where + 1.36 returned a newer one, so the watch DELETED placeholders shift by one; +- `pod/graceful-delete/`: the deleted pod's `PodReadyToStartContainers` + condition now carries `reason: PodSandboxNotReady` and a message; +- `configmap/owner-ref-cascade/`: the garbage collector's `userAgent` names the + new version. + +Every other scenario reproduced unchanged. diff --git a/test/mutationlab/corpus/configmap/deletecollection/audit.deletecollection.yaml b/test/mutationlab/corpus/configmap/deletecollection/audit.deletecollection.yaml index a5c1b50e0..64c07d382 100644 --- a/test/mutationlab/corpus/configmap/deletecollection/audit.deletecollection.yaml +++ b/test/mutationlab/corpus/configmap/deletecollection/audit.deletecollection.yaml @@ -48,7 +48,7 @@ responseObject: uid: kind: ConfigMapList metadata: - resourceVersion: + resourceVersion: responseStatus: code: 200 metadata: {} diff --git a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-a.yaml b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-a.yaml index fde7f8cf3..70f66e10f 100644 --- a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-a.yaml +++ b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-a.yaml @@ -23,6 +23,6 @@ object: time: name: cm-a namespace: - resourceVersion: + resourceVersion: uid: type: DELETED diff --git a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-b.yaml b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-b.yaml index f09f9277b..4ff490827 100644 --- a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-b.yaml +++ b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-b.yaml @@ -23,6 +23,6 @@ object: time: name: cm-b namespace: - resourceVersion: + resourceVersion: uid: type: DELETED diff --git a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-c.yaml b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-c.yaml index 0f7d5d31c..1b4533da3 100644 --- a/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-c.yaml +++ b/test/mutationlab/corpus/configmap/deletecollection/watch.deleted.cm-c.yaml @@ -23,6 +23,6 @@ object: time: name: cm-c namespace: - resourceVersion: + resourceVersion: uid: type: DELETED diff --git a/test/mutationlab/corpus/configmap/deletion-intent-actor/audit.delete.yaml b/test/mutationlab/corpus/configmap/deletion-intent-actor/audit.delete.yaml index 227ddec45..8e50b0961 100644 --- a/test/mutationlab/corpus/configmap/deletion-intent-actor/audit.delete.yaml +++ b/test/mutationlab/corpus/configmap/deletion-intent-actor/audit.delete.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-human" to User "alice@example.com"' validate-all.configbutler.ai/scenario: deletion-intent-actor auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/owner-ref-cascade/audit.delete.cm-child.yaml b/test/mutationlab/corpus/configmap/owner-ref-cascade/audit.delete.cm-child.yaml index f4c7b3c75..b75e1be44 100644 --- a/test/mutationlab/corpus/configmap/owner-ref-cascade/audit.delete.cm-child.yaml +++ b/test/mutationlab/corpus/configmap/owner-ref-cascade/audit.delete.cm-child.yaml @@ -51,5 +51,5 @@ user: - system:authenticated uid: username: system:serviceaccount:kube-system:generic-garbage-collector -userAgent: k3s/v1.36.4+k3s1 (linux/amd64) kubernetes/4dedb15/system:serviceaccount:kube-system:generic-garbage-collector +userAgent: k3s/v1.37.1+k3s1 (linux/amd64) kubernetes/356c025/system:serviceaccount:kube-system:generic-garbage-collector verb: delete diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-exit-deleted.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-exit-deleted.yaml index 9569aa032..41b006e17 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-exit-deleted.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-exit-deleted.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-gone.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-gone.yaml index 070bc4a8d..3d9273bd7 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-gone.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-gone.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-held.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-held.yaml index d0fcdedf1..1293ee9f2 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-held.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-held.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-member.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-member.yaml index c4c34842b..07e70f066 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-member.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-member.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-outsider.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-outsider.yaml index 5136ea131..a9bfcd32f 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-outsider.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.create.cm-outsider.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-exit-deleted.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-exit-deleted.yaml index 072e16b92..7f4c37b90 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-exit-deleted.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-exit-deleted.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-car" to User "carol@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-gone.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-gone.yaml index 8187607d4..e219a7c84 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-gone.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-gone.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-bob" to User "bob@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-held.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-held.yaml index a32843a74..06b82fce7 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-held.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.delete.cm-held.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.1.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.1.yaml index ed9f67e0e..7057f86f7 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.1.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.1.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-bob" to User "bob@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.3.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.3.yaml index ac540d0f0..31c4b9945 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.3.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.3.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.4.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.4.yaml index a9e3591b4..7a7d77e33 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.patch.4.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.patch.4.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-bob" to User "bob@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.update.1.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.update.1.yaml index f3969a5d0..6c427dbdd 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.update.1.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.update.1.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.update.2.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.update.2.yaml index ad5006865..881561c41 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.update.2.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.update.2.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-bob" to User "bob@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/configmap/selector-membership/audit.update.3.yaml b/test/mutationlab/corpus/configmap/selector-membership/audit.update.3.yaml index 73fb87fd6..2afe88ad8 100644 --- a/test/mutationlab/corpus/configmap/selector-membership/audit.update.3.yaml +++ b/test/mutationlab/corpus/configmap/selector-membership/audit.update.3.yaml @@ -4,6 +4,8 @@ annotations: of Role "scenario-ali" to User "alice@example.com"' validate-all.configbutler.ai/scenario: selector-membership auditID: +authenticationMetadata: + impersonationConstraint: impersonate:user-info impersonatedUser: groups: - system:authenticated diff --git a/test/mutationlab/corpus/pod/graceful-delete/watch.deleted.yaml b/test/mutationlab/corpus/pod/graceful-delete/watch.deleted.yaml index 3e588443d..9f1e4b333 100644 --- a/test/mutationlab/corpus/pod/graceful-delete/watch.deleted.yaml +++ b/test/mutationlab/corpus/pod/graceful-delete/watch.deleted.yaml @@ -61,7 +61,9 @@ object: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} + f:message: {} f:observedGeneration: {} + f:reason: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: @@ -131,7 +133,9 @@ object: conditions: - lastProbeTime: null lastTransitionTime: + message: pod sandbox is not ready observedGeneration: 2 + reason: PodSandboxNotReady status: "False" type: PodReadyToStartContainers - lastProbeTime: null diff --git a/test/mutationlab/deploy.sh b/test/mutationlab/deploy.sh new file mode 100644 index 000000000..435bfe097 --- /dev/null +++ b/test/mutationlab/deploy.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Deploy the mutation-capture lab and the wardle sample aggregated API server onto the lab's own +# cluster, and wait until both serve. Idempotent: an unchanged image and certificate set applies +# as a no-op. +# +# Inputs (env): +# - LAB_CLUSTER_NAME (required): k3d cluster name, without the k3d- prefix +# - LAB_CTX (required): its kube context +# - LAB_CS (required): its stamp directory, holding certs/ from test/mutationlab/tools/lab-certs +# - LAB_IMAGE (required): the lab image, already built +# - LAB_NAMESPACE (optional, default mutation-capture-lab) +# - K3D / KUBECTL / CONTAINER_TOOL (optional): tool overrides + +: "${LAB_CLUSTER_NAME:?LAB_CLUSTER_NAME is required}" +: "${LAB_CTX:?LAB_CTX is required}" +: "${LAB_CS:?LAB_CS is required}" +: "${LAB_IMAGE:?LAB_IMAGE is required}" +LAB_NAMESPACE="${LAB_NAMESPACE:-mutation-capture-lab}" +K3D="${K3D:-k3d}" +KUBECTL="${KUBECTL:-kubectl}" +CONTAINER_TOOL="${CONTAINER_TOOL:-docker}" +NODE="k3d-${LAB_CLUSTER_NAME}-server-0" +LAB_DIR="test/mutationlab" +SAMPLE_APISERVER_DIR="test/e2e/setup/manifests/sample-apiserver" + +kubectl_lab() { "${KUBECTL}" --context "${LAB_CTX}" "$@"; } + +image_in_node() { "${CONTAINER_TOOL}" exec "${NODE}" crictl inspecti "${LAB_IMAGE}" >/dev/null 2>&1; } + +# import_image delivers the lab image into the node. `-m direct` streams `docker save` straight +# into the node's `ctr image import`: the default tools-node mode can misread the save as finished +# before the tarball lands and still exit 0. Direct mode can hang instead, so each attempt is +# bounded, and every attempt is checked against the node rather than trusted. +import_image() { + if image_in_node; then + echo "✅ ${LAB_IMAGE} is already in ${NODE}" + return 0 + fi + local attempt + for attempt in 1 2 3; do + echo "📦 Importing ${LAB_IMAGE} into ${LAB_CLUSTER_NAME} (attempt ${attempt}/3)…" + timeout --kill-after=10s 300s "${K3D}" image import -m direct -c "${LAB_CLUSTER_NAME}" "${LAB_IMAGE}" || true + if image_in_node; then + return 0 + fi + done + echo "❌ ${LAB_IMAGE} did not reach ${NODE} after 3 attempts" >&2 + return 1 +} + +# write_overlay renders the overlay that binds the tracked manifests to this cluster: the image +# tag, the Secret with the issued certificates, and the CA as the registration's caBundle. It +# lives in the stamp directory beside certs/, because kustomize reads a generator's files only +# from inside its own root. +write_overlay() { + local base ca_bundle + base="$(realpath --relative-to="${LAB_CS}" "${LAB_DIR}/deploy")" + ca_bundle="$(base64 -w0 <"${LAB_CS}/certs/ca.crt")" + cat >"${LAB_CS}/kustomization.yaml" </dev/null | grep -q flunders; then + return 0 + fi + sleep 2 + done + echo "❌ the wardle APIService is Available, but discovery never listed flunders" >&2 + kubectl_lab get apiservice v1alpha1.wardle.example.com -o yaml >&2 || true + return 1 +} + +import_image +write_overlay +echo "🚀 Applying the lab and the sample aggregated API server…" +kubectl_lab apply -k "${LAB_CS}" +kubectl_lab apply -k "${SAMPLE_APISERVER_DIR}" +kubectl_lab -n "${LAB_NAMESPACE}" rollout status deployment/mutation-capture-lab --timeout=180s +wait_aggregated_api +echo "✅ The lab and the sample aggregated API server are serving." diff --git a/test/mutationlab/deploy/deployment.yaml b/test/mutationlab/deploy/deployment.yaml new file mode 100644 index 000000000..8be92f081 --- /dev/null +++ b/test/mutationlab/deploy/deployment.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mutation-capture-lab + namespace: mutation-capture-lab + labels: + app.kubernetes.io/name: mutation-capture-lab +spec: + # One replica: the records live in this pod's memory, so every audit batch, admission review, + # and watch event has to reach the same store. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: mutation-capture-lab + template: + metadata: + labels: + app.kubernetes.io/name: mutation-capture-lab + spec: + serviceAccountName: mutation-capture-lab + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: lab + # The overlay sets the content-hashed tag; the image is imported into the node, never + # pulled. + image: mutation-capture-lab + imagePullPolicy: Never + args: + - --admission-addr=:9443 + - --admission-cert-dir=/certs + - --audit-addr=:9444 + - --audit-cert-dir=/certs + # The API server presents a client certificate from the lab CA, and the audit + # server refuses a connection without one, as a production receiver does. + - --audit-client-ca=/certs/ca.crt + - --api-addr=:8080 + # configmaps (rows 1-4, 8-13, 16-19), deployments (rows 5, 6), pods (row 7), the + # two-version Widget CRD in its storage version v2 (row 14), and the wardle + # aggregated API (row 15). A watch on a type that does not exist yet retries until + # it does. + - --watch-resources=v1/configmaps,apps/v1/deployments,v1/pods,mutationlab.configbutler.ai/v2/widgets,wardle.example.com/v1alpha1/flunders + ports: + - name: webhook + containerPort: 9443 + - name: audit + containerPort: 9444 + - name: api + containerPort: 8080 + readinessProbe: + httpGet: + path: /readyz + port: api + periodSeconds: 2 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 50m + memory: 64Mi + volumeMounts: + - name: certs + mountPath: /certs + readOnly: true + volumes: + - name: certs + secret: + secretName: mutation-capture-lab-tls diff --git a/test/mutationlab/deploy/kustomization.yaml b/test/mutationlab/deploy/kustomization.yaml new file mode 100644 index 000000000..f12f99b28 --- /dev/null +++ b/test/mutationlab/deploy/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The mutation-capture lab on its own cluster (docs/design/mutation-capture-lab-own-cluster.md). +# `task lab-bootstrap` applies this through an overlay in the lab's stamp directory, which adds +# the content-hashed image tag, the mutation-capture-lab-tls Secret issued by +# test/mutationlab/tools/lab-certs, and that CA as the registration's caBundle. +resources: + - namespace.yaml + - rbac.yaml + - deployment.yaml + - service.yaml + - validating-webhook.yaml diff --git a/test/mutationlab/deploy/namespace.yaml b/test/mutationlab/deploy/namespace.yaml new file mode 100644 index 000000000..8c6992c33 --- /dev/null +++ b/test/mutationlab/deploy/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mutation-capture-lab diff --git a/test/mutationlab/deploy/rbac.yaml b/test/mutationlab/deploy/rbac.yaml new file mode 100644 index 000000000..01b4bb285 --- /dev/null +++ b/test/mutationlab/deploy/rbac.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mutation-capture-lab + namespace: mutation-capture-lab +--- +# Read-only, and only what the Deployment's --watch-resources names: the watch recorder and the +# /watch-probe endpoint open watches, nothing else. A resource added there needs a rule here. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mutation-capture-lab +rules: + - apiGroups: [""] + resources: ["configmaps", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["mutationlab.configbutler.ai"] + resources: ["widgets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["wardle.example.com"] + resources: ["flunders"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: mutation-capture-lab +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mutation-capture-lab +subjects: + - kind: ServiceAccount + name: mutation-capture-lab + namespace: mutation-capture-lab diff --git a/test/mutationlab/deploy/service.yaml b/test/mutationlab/deploy/service.yaml new file mode 100644 index 000000000..13e3ce45c --- /dev/null +++ b/test/mutationlab/deploy/service.yaml @@ -0,0 +1,35 @@ +# Admission reviews, CRD conversion, and the records API. The API server addresses the webhooks +# through this Service, so its DNS name is the one the serving certificate carries. +apiVersion: v1 +kind: Service +metadata: + name: mutation-capture-lab + namespace: mutation-capture-lab +spec: + selector: + app.kubernetes.io/name: mutation-capture-lab + ports: + - name: webhook + port: 443 + targetPort: webhook + - name: api + port: 8080 + targetPort: api +--- +# Audit. The audit webhook backend is configured before the cluster exists, when no Service has a +# ClusterIP yet, so the API server posts to this fixed NodePort on its own node: +# https://127.0.0.1:30444/audit-webhook, verifying the Service's DNS name as tls-server-name. +apiVersion: v1 +kind: Service +metadata: + name: mutation-capture-lab-audit + namespace: mutation-capture-lab +spec: + type: NodePort + selector: + app.kubernetes.io/name: mutation-capture-lab + ports: + - name: audit + port: 9444 + targetPort: audit + nodePort: 30444 diff --git a/test/mutationlab/deploy/validating-webhook.yaml b/test/mutationlab/deploy/validating-webhook.yaml new file mode 100644 index 000000000..0cd484133 --- /dev/null +++ b/test/mutationlab/deploy/validating-webhook.yaml @@ -0,0 +1,36 @@ +# The rules the corpus was captured under; changing them changes what the corpus records. +# +# - resources: ["*"] matches top-level resources only, so rows 5 and 6 record no admission for +# /status and /scale. Kubernetes would send those reviews if a rule named the subresources. +# - failurePolicy Ignore: the backend is a pod on this cluster, and a Fail policy on "*" would +# refuse the writes that bring that pod back. +# - timeoutSeconds 1: with Ignore, a longer wait buys nothing but latency on every write. +# - The webhook's name is part of the capture too: the API server keys the audit annotation the +# admission recorder returns by it (validate-all.configbutler.ai/scenario), and quotes it in a +# rejection's message (row 12). +# +# The overlay `task lab-bootstrap` writes sets caBundle to the lab CA. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: mutation-capture-lab +webhooks: + - name: validate-all.configbutler.ai + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: mutation-capture-lab + namespace: mutation-capture-lab + path: /validate-all + port: 443 + caBundle: "" + failurePolicy: Ignore + matchPolicy: Equivalent + rules: + - apiGroups: ["*"] + apiVersions: ["*"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["*"] + scope: "*" + sideEffects: None + timeoutSeconds: 1 diff --git a/test/mutationlab/e2e/cluster_provenance_test.go b/test/mutationlab/e2e/cluster_provenance_test.go new file mode 100644 index 000000000..cf61f8510 --- /dev/null +++ b/test/mutationlab/e2e/cluster_provenance_test.go @@ -0,0 +1,110 @@ +//go:build mutationlab_e2e + +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "k8s.io/client-go/kubernetes" +) + +// The rows of test/mutationlab/corpus/CLUSTER.md this test owns. The prose around the table is +// left alone. +var ( + imageRow = regexp.MustCompile("(?m)^\\| k3d image \\| `([^`]*)` \\|$") + versionRow = regexp.MustCompile("(?m)^\\| Server version \\| `([^`]*)` \\(([^)]*)\\) \\|$") + capturedRow = regexp.MustCompile(`(?m)^\| Captured at \| ([^|]*) \|$`) +) + +// TestClusterProvenance keeps test/mutationlab/corpus/CLUSTER.md and the cluster that produced the +// corpus in agreement. With MUTATIONLAB_UPDATE it records the cluster's k3s image, server version, +// and the date; otherwise it fails when the cluster runs another version than the corpus was +// captured on, because a corpus diff is then a Kubernetes change, and is accepted with `task +// lab-corpus-update`, not read as a regression. +func TestClusterProvenance(t *testing.T) { + if os.Getenv("LAB_API_URL") == "" { + t.Skip("LAB_API_URL not set; run via `task lab-capture`") + } + image := os.Getenv("LAB_K3S_IMAGE") + if image == "" { + t.Fatal("LAB_K3S_IMAGE not set; run via `task lab-capture`") + } + cfg, err := restConfig() + if err != nil { + t.Fatalf("kube config: %v", err) + } + kube, err := kubernetes.NewForConfig(cfg) + if err != nil { + t.Fatalf("kube client: %v", err) + } + info, err := kube.Discovery().ServerVersion() + if err != nil { + t.Fatalf("server version: %v", err) + } + // rancher/k3s:v1.36.4-k3s1 serves v1.36.4+k3s1. A mismatch means the image recorded in the + // stamps is not what the cluster runs, and writing either into CLUSTER.md would be wrong. + if want := k3sServerVersion(image); want != info.GitVersion { + t.Fatalf("the lab cluster serves %s, but its stamps say it was created from %s", info.GitVersion, image) + } + + path := filepath.Join("..", "corpus", "CLUSTER.md") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + doc := string(raw) + if os.Getenv("MUTATIONLAB_UPDATE") != "" { + updated, err := recordProvenance(doc, image, info.GitVersion, info.Platform, time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(updated), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + t.Logf("recorded %s (%s) in %s", info.GitVersion, image, path) + return + } + m := versionRow.FindStringSubmatch(doc) + if m == nil { + t.Fatalf("%s has no Server version row", path) + } + if m[1] != info.GitVersion { + t.Fatalf("the corpus was captured on %s and the lab cluster serves %s: "+ + "run `task lab-corpus-update` and review the corpus diff as that version's changes", m[1], info.GitVersion) + } +} + +// k3sServerVersion maps a k3s image to the version its API server reports. +func k3sServerVersion(image string) string { + tag := image[strings.LastIndex(image, ":")+1:] + return strings.Replace(tag, "-k3s", "+k3s", 1) +} + +// recordProvenance rewrites the three provenance rows of CLUSTER.md, and fails rather than append +// a row it cannot find, since the table's shape is what `task lab-bootstrap` reads the default +// k3s image from. +func recordProvenance(doc, image, version, platform string, now time.Time) (string, error) { + rows := []struct { + re *regexp.Regexp + row string + }{ + {imageRow, fmt.Sprintf("| k3d image | `%s` |", image)}, + {versionRow, fmt.Sprintf("| Server version | `%s` (%s) |", version, platform)}, + {capturedRow, fmt.Sprintf("| Captured at | %s |", now.Format(time.DateOnly))}, + } + for _, r := range rows { + if !r.re.MatchString(doc) { + return "", fmt.Errorf("CLUSTER.md has no row matching %s", r.re) + } + doc = r.re.ReplaceAllLiteralString(doc, r.row) + } + return doc, nil +} diff --git a/test/mutationlab/e2e/crd_conversion_test.go b/test/mutationlab/e2e/crd_conversion_test.go index f92f6abd5..f18ba1d34 100644 --- a/test/mutationlab/e2e/crd_conversion_test.go +++ b/test/mutationlab/e2e/crd_conversion_test.go @@ -6,9 +6,12 @@ package e2e import ( "context" + "encoding/base64" + "os" "testing" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -37,18 +40,20 @@ var ( Version: "v1", Resource: "customresourcedefinitions", } - vwcGVR = schema.GroupVersionResource{ - Group: "admissionregistration.k8s.io", - Version: "v1", - Resource: "validatingwebhookconfigurations", - } widgetV1 = schema.GroupVersionResource{Group: widgetGroup, Version: "v1", Resource: "widgets"} widgetV2 = schema.GroupVersionResource{Group: widgetGroup, Version: "v2", Resource: "widgets"} - vwcName = "gitops-reverser-validating-webhook" +) + +// The lab's Service, which serves /convert beside /validate-all under the same certificate +// (test/mutationlab/deploy/service.yaml). +const ( + labService = "mutation-capture-lab" + labNamespace = "mutation-capture-lab" + labPort = int64(443) ) // TestCRDConversion captures Row 14. It installs the two-version Widget CRD whose -// conversion webhook reuses the lab's admission cert/port at /convert, creates a +// conversion webhook calls the lab's /convert, creates a // Widget in v1, and captures the divergence: admission + audit see the submitted // v1 (spec.sizeBytes), the watch (opened on the v2 storage version) sees the // stored v2 (spec.size), and the conversion webhook is called in both directions. @@ -111,48 +116,46 @@ func TestCRDConversion(t *testing.T) { } // installWidgetCRD creates the two-version Widget CRD, pointing its conversion -// webhook at the lab by reusing the service and CA bundle the product's -// validating webhook already uses (so the lab needs no new certificate). It waits -// for the CRD to be Established and registers teardown. +// webhook at the lab's Service and trusting the lab CA from the stamp directory +// (LAB_CA_FILE). It waits for the CRD to be Established and registers a teardown +// that waits for the CRD to be gone, so the next install does not meet it still +// terminating. func (h *harness) installWidgetCRD(ctx context.Context, t *testing.T) { t.Helper() - svc, ca := h.admissionServiceAndCA(ctx, t) - crd := widgetCRDObject(svc, ca) + caFile := os.Getenv("LAB_CA_FILE") + if caFile == "" { + t.Fatal("LAB_CA_FILE not set; run via `task lab-capture`") + } + ca, err := os.ReadFile(caFile) + if err != nil { + t.Fatalf("read the lab CA: %v", err) + } + crd := widgetCRDObject(base64.StdEncoding.EncodeToString(ca)) if _, err := h.dyn.Resource(crdGVR).Create(ctx, crd, metav1.CreateOptions{}); err != nil { t.Fatalf("create Widget CRD: %v", err) } - t.Cleanup(func() { - _ = h.dyn.Resource(crdGVR).Delete(context.Background(), widgetCRD, metav1.DeleteOptions{}) - }) + t.Cleanup(func() { h.deleteWidgetCRD(t) }) h.waitForCRDEstablished(ctx, t) } -// admissionServiceAndCA reads the product validating webhook config and returns -// its clientConfig.service (name/namespace/port) and caBundle, so the Widget CRD -// conversion webhook can address the same lab endpoint over the same TLS trust. -func (h *harness) admissionServiceAndCA(ctx context.Context, t *testing.T) (map[string]any, string) { +func (h *harness) deleteWidgetCRD(t *testing.T) { t.Helper() - vwc, err := h.dyn.Resource(vwcGVR).Get(ctx, vwcName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("get validating webhook config %s: %v", vwcName, err) - } - webhooks, _, _ := unstructured.NestedSlice(vwc.Object, "webhooks") - if len(webhooks) == 0 { - t.Fatalf("validating webhook config %s has no webhooks", vwcName) - } - cc, _ := webhooks[0].(map[string]any)["clientConfig"].(map[string]any) - svc, _ := cc["service"].(map[string]any) - ca, _ := cc["caBundle"].(string) - if svc == nil || ca == "" { - t.Fatalf("validating webhook config %s missing service/caBundle", vwcName) + ctx := context.Background() + _ = h.dyn.Resource(crdGVR).Delete(ctx, widgetCRD, metav1.DeleteOptions{}) + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + if _, err := h.dyn.Resource(crdGVR).Get(ctx, widgetCRD, metav1.GetOptions{}); apierrors.IsNotFound(err) { + return + } + time.Sleep(time.Second) } - return svc, ca + t.Errorf("Widget CRD still present 60s after its deletion") } // widgetCRDObject builds the two-version Widget CRD. v1 carries spec.sizeBytes // (integer); v2 (storage) carries spec.size (string); the conversion webhook -// renames between them at the given service/path with the given CA bundle. -func widgetCRDObject(svc map[string]any, ca string) *unstructured.Unstructured { +// renames between them at the lab's /convert, trusting the given CA bundle. +func widgetCRDObject(ca string) *unstructured.Unstructured { intSpec := map[string]any{"type": "object", "properties": map[string]any{ "spec": map[string]any{"type": "object", "properties": map[string]any{ "sizeBytes": map[string]any{"type": "integer"}}}}} @@ -160,10 +163,10 @@ func widgetCRDObject(svc map[string]any, ca string) *unstructured.Unstructured { "spec": map[string]any{"type": "object", "properties": map[string]any{ "size": map[string]any{"type": "string"}}}}} convService := map[string]any{ - "name": svc["name"], - "namespace": svc["namespace"], + "name": labService, + "namespace": labNamespace, "path": "/convert", - "port": svc["port"], + "port": labPort, } return &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "apiextensions.k8s.io/v1", diff --git a/test/mutationlab/e2e/doc.go b/test/mutationlab/e2e/doc.go index 4a7e8ccc7..2111e0e24 100644 --- a/test/mutationlab/e2e/doc.go +++ b/test/mutationlab/e2e/doc.go @@ -7,5 +7,6 @@ // The driver is behind the `mutationlab_e2e` build tag so it never compiles into // the default `go test ./...` unit lane (which has no cluster). This file is the // untagged placeholder that keeps the package buildable when the tag is absent; -// `task lab-e2e` builds it with -tags mutationlab_e2e. +// `task lab-capture` builds it with -tags mutationlab_e2e against the lab's own +// cluster (test/mutationlab/Taskfile-lab.yml). package e2e diff --git a/test/mutationlab/e2e/harness_test.go b/test/mutationlab/e2e/harness_test.go index 7fb82f682..f0c5f88f2 100644 --- a/test/mutationlab/e2e/harness_test.go +++ b/test/mutationlab/e2e/harness_test.go @@ -54,7 +54,12 @@ func newHarness(t *testing.T) *harness { t.Helper() apiURL := os.Getenv("LAB_API_URL") if apiURL == "" { - t.Skip("LAB_API_URL not set; run via `task lab-e2e`") + t.Skip("LAB_API_URL not set; run via `task lab-capture`") + } + // The context is required, never defaulted: the current context may well be the e2e + // cluster, and the scenarios would then write there while the lab records nothing. + if os.Getenv("LAB_KUBECONTEXT") == "" { + t.Fatal("LAB_KUBECONTEXT not set; run via `task lab-capture`") } cfg, err := restConfig() if err != nil { @@ -256,10 +261,10 @@ func (h *harness) createNamespace(ctx context.Context, t *testing.T, ns string) } // ensureCaptureLive creates a labeled probe ConfigMap and waits until its -// admission record reaches the lab, confirming the webhook endpoint has finished -// cutting over to the lab pod after the image swap (the apiserver routes to the -// Service, whose endpoints lag a rollout). Without this, the first real scenario -// can silently miss its admission record. +// admission record reaches the lab, confirming the webhook Service routes to a +// ready lab pod (its endpoints lag a rollout). Without this, the first real +// scenario can silently miss its admission record. TestSourcesLive, which +// `task lab-bootstrap` runs, proves every other source. func (h *harness) ensureCaptureLive(t *testing.T) { t.Helper() ctx := context.Background() @@ -275,7 +280,7 @@ func (h *harness) ensureCaptureLive(t *testing.T) { return } } - t.Fatal("capture never went live: no admission record within 90s of the image swap") + t.Fatal("capture never went live: no admission record within 90s") } func hasSource(records []mutationlab.Record, src mutationlab.Source) bool { @@ -413,9 +418,8 @@ func (h *harness) probeWatch(t *testing.T, req watchProbeRequest) []mutationlab. } // do issues an HTTP request to the lab API, retrying transient connection errors. -// The API sits behind a kubectl port-forward that can drop and is restarted by a -// watchdog in the lab-e2e task, so a brief connection failure is expected, not -// fatal. +// The API sits behind a kubectl port-forward that can drop, so a brief connection +// failure is expected, not fatal. func (h *harness) do(method, path string) ([]byte, int, error) { return h.doJSON(method, path, nil) } diff --git a/test/mutationlab/e2e/sources_live_test.go b/test/mutationlab/e2e/sources_live_test.go new file mode 100644 index 000000000..2dc559350 --- /dev/null +++ b/test/mutationlab/e2e/sources_live_test.go @@ -0,0 +1,122 @@ +//go:build mutationlab_e2e + +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/ConfigButler/gitops-reverser/internal/mutationlab" +) + +// TestSourcesLive proves that every source the corpus records reaches the lab, each by a probe +// write it then finds among the records: the audit webhook backend, the admission webhook, the +// CRD conversion webhook, the aggregated API server, and the watch. `task lab-bootstrap` runs it +// alone, so a capture starts only on a cluster where all five deliver; the capture itself skips +// it. +// +// It exists because a source can go quiet without anything failing: the swapped-in lab of old +// lost its admission registration, and audit kept flowing, so only a check that names each source +// tells a missing one apart from a scenario that has none. +func TestSourcesLive(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + + t.Run("audit, admission, and watch", func(t *testing.T) { + s := h.newScenario(ctx, t, "live-configmap") + cm := &corev1.ConfigMap{ObjectMeta: s.meta("probe"), Data: map[string]string{"k": "v"}} + if _, err := h.kube.CoreV1().ConfigMaps(s.ns).Create(ctx, cm, metav1.CreateOptions{}); err != nil { + t.Fatalf("create probe ConfigMap: %v", err) + } + h.awaitSources(t, s.id, "", []sourceCheck{ + {"audit", func(rs []mutationlab.Record) bool { return firstSource(rs, mutationlab.SourceAudit) != nil }}, + {"admission", func(rs []mutationlab.Record) bool { + return firstSource(rs, mutationlab.SourceAdmission) != nil + }}, + {"watch", func(rs []mutationlab.Record) bool { return firstWatch(rs, "ADDED") != nil }}, + }) + }) + + t.Run("conversion", func(t *testing.T) { + h.installWidgetCRD(ctx, t) + s := h.newScenario(ctx, t, "live-conversion") + h.ensureWidgetWatchLive(ctx, t, s) + w := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": widgetGroup + "/v1", + "kind": "Widget", + "metadata": map[string]any{ + "name": "probe-2", "namespace": s.ns, "labels": map[string]any{scenarioLabel: s.id}, + }, + "spec": map[string]any{"sizeBytes": int64(1)}, + }} + if _, err := h.dyn.Resource(widgetV1).Namespace(s.ns).Create(ctx, w, metav1.CreateOptions{}); err != nil { + t.Fatalf("create probe Widget: %v", err) + } + h.awaitSources(t, s.id, "", []sourceCheck{ + {"conversion", func(rs []mutationlab.Record) bool { + return firstSource(rs, mutationlab.SourceConversion) != nil + }}, + }) + }) + + t.Run("aggregated API", func(t *testing.T) { + s := h.newScenario(ctx, t, "live-aggregated") + fl := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "wardle.example.com/v1alpha1", + "kind": "Flunder", + "metadata": map[string]any{ + "name": "probe", "namespace": s.ns, "labels": map[string]any{scenarioLabel: s.id}, + }, + "spec": map[string]any{"referenceType": "Flunder", "reference": "probe"}, + }} + if _, err := h.dyn.Resource(flunderGVR).Namespace(s.ns).Create(ctx, fl, metav1.CreateOptions{}); err != nil { + t.Fatalf("create probe flunder: %v", err) + } + // The namespace key as well: an aggregated API's audit event has no body to carry the + // scenario label, and its objectRef has no name for a create (row 15). + h.awaitSources(t, s.id, s.ns, []sourceCheck{ + {"aggregated API watch", func(rs []mutationlab.Record) bool { + return flunderRecordNamed(rs, mutationlab.SourceWatch, "ADDED", "probe") != nil + }}, + {"aggregated API audit", func(rs []mutationlab.Record) bool { + return flunderRecordNamed(rs, mutationlab.SourceAudit, "", "") != nil + }}, + }) + }) +} + +type sourceCheck struct { + name string + found func([]mutationlab.Record) bool +} + +// awaitSources polls the records of one scenario, unioned with those of namespace when it is set, +// until every check finds its record, and on timeout names the sources that never arrived. +func (h *harness) awaitSources(t *testing.T, id, namespace string, checks []sourceCheck) { + t.Helper() + const timeout = 90 * time.Second + deadline := time.Now().Add(timeout) + var missing []string + for time.Now().Before(deadline) { + records := h.fetchScenario(t, id, namespace) + missing = missing[:0] + for _, c := range checks { + if !c.found(records) { + missing = append(missing, c.name) + } + } + if len(missing) == 0 { + return + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("no %s record reached the lab within %s", strings.Join(missing, ", "), timeout) +} diff --git a/test/mutationlab/e2e/workload_scenarios_test.go b/test/mutationlab/e2e/workload_scenarios_test.go index 7010c613a..946ea69e3 100644 --- a/test/mutationlab/e2e/workload_scenarios_test.go +++ b/test/mutationlab/e2e/workload_scenarios_test.go @@ -22,9 +22,10 @@ import ( // Deployment for the /status (Row 5) and /scale (Row 6) subresources, and a Pod // for the two-event graceful delete (Row 7). // -// The headline finding here is mechanism *silence*. The lab reuses the product's -// audit policy and validating-webhook config verbatim (swap-image model), and -// those deliberately drop exactly these moments from the provenance mechanisms: +// The headline finding here is mechanism *silence*. The lab cluster uses the +// e2e audit policy and the validating-webhook match rules the corpus was captured +// under, and those deliberately drop exactly these moments from the provenance +// mechanisms: // // - the audit policy drops apps/*/status and core */status, so a /status write // is invisible to audit (Row 5); diff --git a/test/mutationlab/port-forward.sh b/test/mutationlab/port-forward.sh new file mode 100644 index 000000000..f9c535ebb --- /dev/null +++ b/test/mutationlab/port-forward.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Forward a host port to the mutation-capture lab's records API, detached with setsid so it +# outlives the task that starts it, and verified healthy before returning. +# +# A forward left from an earlier run is always replaced: it points at the pod that was current +# then, and a new image rolls a new pod. Only this cluster's forward is touched, so the e2e +# cluster's forwards keep running beside it. +# +# Inputs (env): +# - LAB_CTX (required): the lab's kube context +# - LAB_CS (required): its stamp directory, where the forward's log goes +# - LAB_NAMESPACE (optional, default mutation-capture-lab) +# - LAB_API_PORT (optional, default 18081): the host port +# - KUBECTL (optional): tool override + +: "${LAB_CTX:?LAB_CTX is required}" +: "${LAB_CS:?LAB_CS is required}" +LAB_NAMESPACE="${LAB_NAMESPACE:-mutation-capture-lab}" +LAB_API_PORT="${LAB_API_PORT:-18081}" +KUBECTL="${KUBECTL:-kubectl}" +REMOTE_PORT=8080 + +healthy() { curl -fsS "http://127.0.0.1:${LAB_API_PORT}/healthz" >/dev/null 2>&1; } + +pkill -f "port-forward --context ${LAB_CTX} .* ${LAB_API_PORT}:${REMOTE_PORT}" 2>/dev/null || true +sleep 1 + +"${KUBECTL}" --context "${LAB_CTX}" -n "${LAB_NAMESPACE}" rollout status deployment/mutation-capture-lab \ + --timeout=120s >/dev/null + +echo "🔌 Forwarding 127.0.0.1:${LAB_API_PORT} to the lab's records API…" +mkdir -p "${LAB_CS}" +pf_log="${LAB_CS}/port-forward.log" +# The context comes first so the pkill pattern above matches this cluster's forward only. +setsid "${KUBECTL}" port-forward --context "${LAB_CTX}" -n "${LAB_NAMESPACE}" --address 127.0.0.1 \ + deployment/mutation-capture-lab "${LAB_API_PORT}:${REMOTE_PORT}" >"${pf_log}" 2>&1 /dev/null; then + echo "❌ the lab port-forward exited:" >&2 + sed 's/^/ /' "${pf_log}" >&2 || true + exit 1 + fi + if healthy; then + echo "✅ The lab's records API answers on http://127.0.0.1:${LAB_API_PORT}" + exit 0 + fi + sleep 1 +done + +echo "❌ the lab port-forward was not healthy after 15s:" >&2 +sed 's/^/ /' "${pf_log}" >&2 || true +exit 1 diff --git a/test/mutationlab/selfcontained_test.go b/test/mutationlab/selfcontained_test.go new file mode 100644 index 000000000..585035801 --- /dev/null +++ b/test/mutationlab/selfcontained_test.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package mutationlab holds the mutation-capture lab's cluster wiring. This test keeps that +// wiring self-contained: the lab once borrowed the product's admission registration, certificate, +// and Deployment, and lost its admission source when the product removed them, with every check +// still green. +package mutationlab + +import ( + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "sigs.k8s.io/yaml" +) + +const repoRoot = "../.." + +// sharedPaths are the only files outside the lab's own directories that the lab's wiring may use. +// Each is shared on purpose, and changing one changes both clusters the same way. +var sharedPaths = []string{ + "test/e2e/cluster/start-cluster.sh", // creates both clusters + "test/e2e/cluster/audit/policy.yaml", // one audit policy, so the corpus shows what e2e audits + "test/e2e/setup/manifests/sample-apiserver", // the same aggregated API server + "test/mutationlab", // the lab's wiring + "cmd/mutation-capture-lab", "internal/mutationlab", // the lab binary +} + +// labFiles are the files the check reads: the lab binary and its wiring, without the corpus +// (data the API server wrote) and the Markdown (prose about the product is fine). +func labFiles(t *testing.T) []string { + t.Helper() + var files []string + for _, dir := range []string{"test/mutationlab", "cmd/mutation-capture-lab", "internal/mutationlab"} { + err := filepath.WalkDir(filepath.Join(repoRoot, dir), func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && d.Name() == "corpus" { + return filepath.SkipDir + } + if d.IsDir() || strings.HasSuffix(path, ".md") || strings.HasSuffix(path, "selfcontained_test.go") { + return nil + } + files = append(files, path) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + } + return files +} + +// TestLabTasksDependOnLabTasksOnly fails when a lab task runs or depends on a task the lab's +// Taskfile does not define, which is how the lab came to depend on the product's install. +func TestLabTasksDependOnLabTasksOnly(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(repoRoot, "test/mutationlab/Taskfile-lab.yml")) + if err != nil { + t.Fatal(err) + } + var tf struct { + Tasks map[string]struct { + Deps []any `json:"deps"` + Cmds []any `json:"cmds"` + } `json:"tasks"` + } + if err := yaml.Unmarshal(raw, &tf); err != nil { + t.Fatalf("parse Taskfile-lab.yml: %v", err) + } + for name, task := range tf.Tasks { + for _, ref := range append(taskRefs(task.Deps, true), taskRefs(task.Cmds, false)...) { + if _, ok := tf.Tasks[ref]; !ok { + t.Errorf("lab task %s calls %s, which Taskfile-lab.yml does not define", name, ref) + } + } + } +} + +// taskRefs returns the task names a deps or cmds list calls. A deps entry is a name or a +// {task: name} map; a cmds entry calls a task only in the map form. +func taskRefs(entries []any, bareNames bool) []string { + var refs []string + for _, e := range entries { + switch v := e.(type) { + case string: + if bareNames { + refs = append(refs, v) + } + case map[string]any: + if name, ok := v["task"].(string); ok { + refs = append(refs, name) + } + } + } + return refs +} + +var repoPath = regexp.MustCompile(`\b(?:test|hack|config|charts|cmd|internal|api)/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)*`) + +// TestLabUsesOnlySharedPaths fails when the lab's wiring (its Taskfile, manifests, and scripts) +// reads a repository path outside its own directories and the few it shares on purpose. +func TestLabUsesOnlySharedPaths(t *testing.T) { + for _, file := range labFiles(t) { + if strings.HasSuffix(file, ".go") { + // Go names packages, not files; the compiler and the name check below cover it. + continue + } + raw, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + for _, p := range uniq(repoPath.FindAllString(string(raw), -1)) { + if !shared(p) { + t.Errorf("%s uses %s, outside the lab and the paths it shares (sharedPaths)", rel(file), p) + } + } + } +} + +func shared(p string) bool { + for _, s := range sharedPaths { + if p == s || strings.HasPrefix(p, s+"/") { + return true + } + } + return false +} + +// TestLabNamesNoProductResource fails when the lab names something the product installs: its +// namespace, Deployment, Services, certificates, webhook registrations, or audit routes all carry +// the gitops-reverser name. Two spellings of it are the lab's own: the Go module path, and the lab +// cluster's name. +func TestLabNamesNoProductResource(t *testing.T) { + allowed := strings.NewReplacer( + "github.com/ConfigButler/gitops-reverser", "", + "gitops-reverser-capture-lab", "", + ) + for _, file := range labFiles(t) { + raw, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(raw), "\n") { + if strings.Contains(allowed.Replace(line), "gitops-reverser") { + t.Errorf("%s:%d names a product resource: %s", rel(file), i+1, strings.TrimSpace(line)) + } + } + } +} + +func uniq(in []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +func rel(path string) string { + r, err := filepath.Rel(repoRoot, path) + if err != nil { + return path + } + return r +} diff --git a/test/mutationlab/swap-image.sh b/test/mutationlab/swap-image.sh deleted file mode 100755 index 2f2e38880..000000000 --- a/test/mutationlab/swap-image.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Swap the deployed gitops-reverser controller to the mutation-capture lab image. -# -# The lab deliberately reuses the product's audit + admission webhook wiring (same -# URLs, same TLS cert mounts), so making the cluster capture with the lab instead -# of the controller is just this: import the lab image and patch the existing -# Deployment's image + entrypoint + args. No new audit policy, webhook config, or -# certificates are introduced — that is the whole point of the swap. -# -# Inputs (env): -# - CTX (required): kube context (k3d-) -# - CLUSTER_NAME (required): k3d cluster name (without the k3d- prefix) -# - NAMESPACE (required): controller namespace -# - LAB_IMAGE (required): lab image reference to import and run -# - K3D / KUBECTL (optional): tool overrides -# - CONTROLLER_DEPLOY_SELECTOR (optional): label selector for the controller Deployment -# - CONTROLLER_CONTAINER (optional): container name to patch (default: manager) - -: "${CTX:?CTX is required}" -: "${CLUSTER_NAME:?CLUSTER_NAME is required}" -: "${NAMESPACE:?NAMESPACE is required}" -: "${LAB_IMAGE:?LAB_IMAGE is required}" - -K3D="${K3D:-k3d}" -KUBECTL="${KUBECTL:-kubectl}" -SELECTOR="${CONTROLLER_DEPLOY_SELECTOR:-app.kubernetes.io/part-of=gitops-reverser}" -CONTAINER="${CONTROLLER_CONTAINER:-manager}" - -echo "Importing ${LAB_IMAGE} into k3d cluster ${CLUSTER_NAME}…" -"${K3D}" image import "${LAB_IMAGE}" -c "${CLUSTER_NAME}" - -deploy="$("${KUBECTL}" --context "${CTX}" -n "${NAMESPACE}" get deploy -l "${SELECTOR}" -o name | head -n1)" -[ -n "${deploy}" ] || { echo "no controller Deployment matching ${SELECTOR} in ${NAMESPACE}" >&2; exit 1; } -echo "Patching ${deploy} container '${CONTAINER}' to the lab image…" - -# Strategic-merge patch: container list is keyed by name, command/args replace -# wholesale. The cert mounts and the /readyz:8081 readiness probe are inherited -# from the product Deployment unchanged. -read -r -d '' patch <