diff --git a/.coverage-baseline b/.coverage-baseline index f32b65c84..a03b49422 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -80.3 +82.5 diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json index f4dfa8826..c14fe30d4 100644 --- a/.devcontainer/devcontainer-lock.json +++ b/.devcontainer/devcontainer-lock.json @@ -1,9 +1,9 @@ { "features": { - "ghcr.io/devcontainers/features/common-utils:2.7.0": { - "version": "2.7.0", - "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:3f63bbee7418dc940d9e5e7ab93ef3049acfe4dba3085b17ee45fe307fdba4e5", - "integrity": "sha256:3f63bbee7418dc940d9e5e7ab93ef3049acfe4dba3085b17ee45fe307fdba4e5" + "ghcr.io/devcontainers/features/common-utils:2.7.1": { + "version": "2.7.1", + "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:e76a6a8a35d1ad824daad1540725e0ec9f8ed2dc00d2850e990a9830013c7d08", + "integrity": "sha256:e76a6a8a35d1ad824daad1540725e0ec9f8ed2dc00d2850e990a9830013c7d08" }, "ghcr.io/devcontainers/features/docker-outside-of-docker:1.10.1": { "version": "1.10.1", diff --git a/.github/RELEASES.md b/.github/RELEASES.md index d44535926..b11b51c5f 100644 --- a/.github/RELEASES.md +++ b/.github/RELEASES.md @@ -88,7 +88,8 @@ Push to main 4. **When Release PR is Merged**: - GitHub Release created with tag (e.g., `v0.2.0`) — first as a **draft**, so every signed - asset (`crds.yaml`, `install.yaml`, SBOM, `.sigstore.json` signatures, `.intoto.jsonl` attestations) + asset (`crds.yaml`, `commitrequest-crd.yaml`, `install.yaml`, SBOM, `.sigstore.json` signatures, + `.intoto.jsonl` attestations) can be attached before it goes public; immutable releases reject post-publish uploads, so `publish-release` flips the draft to published only after every asset is in place. - The linux/amd64 + linux/arm64 image digests already built and scanned by that diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c65832e31..64ceaac0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -458,6 +458,7 @@ jobs: name: release-bundle path: | dist/crds.yaml + dist/commitrequest-crd.yaml dist/install.yaml gitops-reverser.tgz if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a60adb27e..cd10cff00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -354,6 +354,11 @@ jobs: - name: Sign crds.yaml (cosign keyless) run: cosign sign-blob --bundle crds.yaml.sigstore.json --yes dist/crds.yaml + # The CommitRequest CRD alone, for the source clusters that serve CommitRequests and need + # none of the other kinds. It is part of crds.yaml too; this copy is signed on its own. + - name: Sign commitrequest-crd.yaml (cosign keyless) + run: cosign sign-blob --bundle commitrequest-crd.yaml.sigstore.json --yes dist/commitrequest-crd.yaml + - name: Attest SLSA provenance for install.yaml id: attest-install-yaml uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 @@ -372,6 +377,15 @@ jobs: - name: Rename crds provenance bundle to the Scorecard-recognized suffix run: cp "${{ steps.attest-crds-yaml.outputs.bundle-path }}" crds.yaml.intoto.jsonl + - name: Attest SLSA provenance for commitrequest-crd.yaml + id: attest-commitrequest-crd + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: dist/commitrequest-crd.yaml + + - name: Rename commitrequest-crd provenance bundle to the Scorecard-recognized suffix + run: cp "${{ steps.attest-commitrequest-crd.outputs.bundle-path }}" commitrequest-crd.yaml.intoto.jsonl + - name: Upload installer bundle as release assets uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: @@ -384,6 +398,9 @@ jobs: dist/crds.yaml crds.yaml.sigstore.json crds.yaml.intoto.jsonl + dist/commitrequest-crd.yaml + commitrequest-crd.yaml.sigstore.json + commitrequest-crd.yaml.intoto.jsonl dist/install.yaml install.yaml.sigstore.json install.yaml.intoto.jsonl diff --git a/.gitignore b/.gitignore index d2f1bdfde..5eb73908d 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,4 @@ loadtest /external-sources/ /docs/local-only/* hack/release-notes/node_modules +docs/review/* diff --git a/AGENTS.md b/AGENTS.md index d3f6af75a..7a66c73fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,9 +96,9 @@ See [Documentation checks](./CONTRIBUTING.md#documentation-checks). ### Webhook Code (`internal/webhook/`) -- Implement admission webhook interface correctly -- Add proper validation/mutation logic -- Update webhook configuration in `config/webhook/` +- It is the audit ingress and the push receiver: HTTP handlers for what kube-apiserver and Git + hosts post. GitOps Reverser ships no admission webhook; validation lives in the schema, CEL and + the reconciler (`docs/spec/where-validation-lives.md`) ### API Changes (`api/v1alpha3/`) diff --git a/README.md b/README.md index 715828459..b7e4149b6 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ commits. If both touch the same resource, the API wins Follow the [installation walkthrough](docs/quickstart.md) to capture ConfigMaps from a demo namespace into a disposable repo. You need a Kubernetes cluster, `kubectl`, Helm 3, and Git write access. -The walkthrough includes cert-manager setup; Redis and audit delivery are optional. +Redis, audit delivery and cert-manager are optional: they come in with named commit authors. The demo captures a live change: diff --git a/SECURITY.md b/SECURITY.md index 9b419074c..125b5eda4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -58,7 +58,7 @@ Security reports are especially relevant for areas such as: - queue and Valkey/Redis handling - Git credential handling - SOPS and Secret processing -- admission webhook validation +- `CommitRequest` admission and serving, including on source clusters ## Supported versions diff --git a/Taskfile-build.yml b/Taskfile-build.yml index 676902cb3..f9a773844 100644 --- a/Taskfile-build.yml +++ b/Taskfile-build.yml @@ -771,6 +771,7 @@ tasks: - exclude: cmd/**/zz_generated.deepcopy.go generates: - dist/crds.yaml + - dist/commitrequest-crd.yaml - dist/install.yaml cmds: - | @@ -784,6 +785,8 @@ tasks: # here. Each chart CRD file already begins with its own `---`, so a plain cat is valid # multi-doc YAML. cat charts/gitops-reverser/crds/*.yaml > dist/crds.yaml + # A source cluster needs only the CommitRequest CRD, so it ships on its own too. + cp charts/gitops-reverser/crds/configbutler.ai_commitrequests.yaml dist/commitrequest-crd.yaml {{.HELM}} template {{.INSTALL_NAME}} charts/gitops-reverser \ --namespace {{.NAMESPACE}} \ --set labels.managedBy=kubectl \ diff --git a/api/v1alpha3/access_grants.go b/api/v1alpha3/access_grants.go index 8518a3361..c0629c0a5 100644 --- a/api/v1alpha3/access_grants.go +++ b/api/v1alpha3/access_grants.go @@ -219,7 +219,72 @@ type KubeProviderApplierClaimGrant struct { ApplierNamespaces []ScopeNamespace `json:"applierNamespaces"` } -// The API server defaults an omitted grant list on GitBranch and KubeProvider (the +// GitTargetCommitRequestGrant admits the CommitRequests of some namespaces to +// save a GitTarget's commit windows: on the cluster GitOps Reverser runs in +// (consumerNamespaces), or on the cluster the GitTarget mirrors +// (sourceNamespaces). It sets exactly one of the two. +// +kubebuilder:validation:XValidation:rule="has(self.consumerNamespaces) != has(self.sourceNamespaces)",message="a grant in allowedCommitRequests sets exactly one of consumerNamespaces and sourceNamespaces" +type GitTargetCommitRequestGrant struct { + // ConsumerNamespaces are the namespaces, on the cluster GitOps Reverser + // runs in, whose CommitRequests this grant admits. from: "Same" is this + // GitTarget's namespace. It admits requests only for a GitTarget whose + // KubeProvider sets spec.inCluster. + // +optional + ConsumerNamespaces *ConsumerNamespaces `json:"consumerNamespaces,omitempty"` + + // SourceNamespaces are the namespaces, on the cluster spec.kubeProviderRef + // names, whose CommitRequests this grant admits: exact names, or "*" for + // every namespace this GitTarget may mirror. Variables are refused. A + // namespace the KubeProvider does not lend this GitTarget in + // spec.allowedGitTargets is never admitted. It admits requests only for a + // GitTarget whose KubeProvider names a kubeconfig. + // +optional + // +listType=set + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + SourceNamespaces []SourceNamespace `json:"sourceNamespaces,omitempty"` +} + +// AdmitsCommitRequest reports whether spec.allowedCommitRequests admits a CommitRequest in namespace +// of the control cluster, by the labels that namespace carries there: its consumerNamespaces grants. +func (t *GitTarget) AdmitsCommitRequest(namespace string, nsLabels map[string]string) (bool, error) { + admitting, err := admitAll(consumersOf(t.commitRequestGrants()), t.Namespace, namespace, nsLabels) + return len(admitting) > 0, err +} + +// DescribeAllowedCommitRequests names the namespaces spec.allowedCommitRequests admits on the control +// cluster. +func (t *GitTarget) DescribeAllowedCommitRequests() string { + grants := t.commitRequestGrants() + consumers := make([]ConsumerNamespaces, 0, len(grants)) + for i := range grants { + if grants[i].ConsumerNamespaces != nil { + consumers = append(consumers, *grants[i].ConsumerNamespaces) + } + } + return describeAdmitted(consumers, t.Namespace) +} + +// AdmitsSourceCommitRequest reports whether spec.allowedCommitRequests admits a CommitRequest in +// namespace of the cluster this GitTarget mirrors: its sourceNamespaces grants, "*" for any. +func (t *GitTarget) AdmitsSourceCommitRequest(namespace string) bool { + for _, grant := range t.commitRequestGrants() { + if slices.ContainsFunc(grant.SourceNamespaces, func(ns SourceNamespace) bool { + return ns == "*" || string(ns) == namespace + }) { + return true + } + } + return false +} + +// SourceNamespace is a namespace of a source cluster, by exact name, or "*" +// for every namespace. +// +kubebuilder:validation:MaxLength=63 +// +kubebuilder:validation:XValidation:rule="self == '*' || self.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')",message="a source namespace is a namespace name, or '*' alone; variables are refused, because a source cluster's namespaces are not the control cluster's" +type SourceNamespace string + +// The API server defaults an omitted grant list on GitBranch, KubeProvider and GitTarget (the // +kubebuilder:default markers on their fields), so a list read from it is never nil. These mirror // those markers for an object that did not come from the API server, such as one built in memory or // read through a fake client: nil is the default, and only a non-nil empty list admits nobody. @@ -249,6 +314,12 @@ func DefaultKubeProviderApplierClaimGrants() []KubeProviderApplierClaimGrant { }} } +// DefaultGitTargetCommitRequestGrants is GitTarget.spec.allowedCommitRequests when it is omitted: +// CommitRequests in the GitTarget's own namespace. +func DefaultGitTargetCommitRequestGrants() []GitTargetCommitRequestGrant { + return []GitTargetCommitRequestGrant{{ConsumerNamespaces: &ConsumerNamespaces{From: GrantFromSame}}} +} + // orDefault is grants, or def when grants is nil (omitted rather than empty). func orDefault[G any](grants []G, def func() []G) []G { if grants == nil { @@ -272,6 +343,20 @@ func (p *KubeProvider) applierClaimGrants() []KubeProviderApplierClaimGrant { return orDefault(p.Spec.AllowedApplierClaims, DefaultKubeProviderApplierClaimGrants) } +// commitRequestGrants is spec.allowedCommitRequests as the API server serves it. +func (t *GitTarget) commitRequestGrants() []GitTargetCommitRequestGrant { + return orDefault(t.Spec.AllowedCommitRequests, DefaultGitTargetCommitRequestGrants) +} + +// consumers is the admitting half of a GitTargetCommitRequestGrant on the control cluster. A grant of +// source namespaces admits nobody there: its zero ConsumerNamespaces has no from. +func (g *GitTargetCommitRequestGrant) consumers() ConsumerNamespaces { + if g.ConsumerNamespaces == nil { + return ConsumerNamespaces{} + } + return *g.ConsumerNamespaces +} + // consumers is the admitting half of a GitProviderGitBranchGrant. func (g *GitProviderGitBranchGrant) consumers() ConsumerNamespaces { return g.ConsumerNamespaces } diff --git a/api/v1alpha3/access_grants_defaults_test.go b/api/v1alpha3/access_grants_defaults_test.go index affd4dc94..58332ce46 100644 --- a/api/v1alpha3/access_grants_defaults_test.go +++ b/api/v1alpha3/access_grants_defaults_test.go @@ -42,6 +42,10 @@ func TestGrantDefaultsMatchCRDs(t *testing.T) { assertCRDDefault(t, "configbutler.ai_kubeproviders.yaml", "allowedApplierClaims", DefaultKubeProviderApplierClaimGrants()) }) + t.Run("GitTarget allowedCommitRequests", func(t *testing.T) { + assertCRDDefault(t, "configbutler.ai_gittargets.yaml", "allowedCommitRequests", + DefaultGitTargetCommitRequestGrants()) + }) } // assertCRDDefault asserts that the CRD's default for spec. decodes to want. @@ -73,6 +77,12 @@ func TestGrantLists_EmptySurvivesSerialization(t *testing.T) { } } + raw, err := json.Marshal(GitTargetSpec{AllowedCommitRequests: []GitTargetCommitRequestGrant{}}) + require.NoError(t, err) + var targetFields map[string]json.RawMessage + require.NoError(t, json.Unmarshal(raw, &targetFields)) + assert.JSONEq(t, "[]", string(targetFields["allowedCommitRequests"]), "GitTarget spec.allowedCommitRequests") + var decoded GitBranchSpec require.NoError(t, json.Unmarshal([]byte(`{"allowedGitTargets":[]}`), &decoded)) assert.NotNil(t, decoded.AllowedGitTargets, "a decoded [] stays distinguishable from omitted") diff --git a/api/v1alpha3/access_grants_test.go b/api/v1alpha3/access_grants_test.go index cf3fc49cf..b921ec50e 100644 --- a/api/v1alpha3/access_grants_test.go +++ b/api/v1alpha3/access_grants_test.go @@ -349,3 +349,33 @@ func TestGitBranch_GitProviderKey(t *testing.T) { branch.Spec.GitProviderRef.Namespace = "platform" assert.Equal(t, "platform/shared-repo", branch.GitProviderKey().String()) } + +// A GitTarget admits a CommitRequest by the side its grant names: consumerNamespaces admit a +// namespace of the control cluster, sourceNamespaces a namespace of the cluster the GitTarget mirrors. +// A grant for one side admits nothing on the other. +func TestGitTarget_AdmitsCommitRequestBySide(t *testing.T) { + target := func(grants ...GitTargetCommitRequestGrant) *GitTarget { + return &GitTarget{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a"}, Spec: GitTargetSpec{ + AllowedCommitRequests: grants, + }} + } + source := func(namespaces ...SourceNamespace) GitTargetCommitRequestGrant { + return GitTargetCommitRequestGrant{SourceNamespaces: namespaces} + } + + defaulted := &GitTarget{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a"}} + admitted, err := defaulted.AdmitsCommitRequest("team-a", nil) + require.NoError(t, err) + assert.True(t, admitted, "the default admits the GitTarget's own namespace") + assert.False(t, defaulted.AdmitsSourceCommitRequest("team-a"), "the default admits nothing on a source cluster") + + assert.False(t, target().AdmitsSourceCommitRequest("shop"), "an empty list admits nobody") + assert.True(t, target(source("*")).AdmitsSourceCommitRequest("shop")) + assert.True(t, target(source("shop")).AdmitsSourceCommitRequest("shop")) + assert.False(t, target(source("cart")).AdmitsSourceCommitRequest("shop")) + + admitted, err = target(source("*")).AdmitsCommitRequest("team-a", nil) + require.NoError(t, err) + assert.False(t, admitted, "a source grant admits nothing on the control cluster") + assert.Equal(t, "admits no namespace", target(source("*")).DescribeAllowedCommitRequests()) +} diff --git a/api/v1alpha3/applierclaim_types.go b/api/v1alpha3/applierclaim_types.go index 5b678f171..cf997c090 100644 --- a/api/v1alpha3/applierclaim_types.go +++ b/api/v1alpha3/applierclaim_types.go @@ -3,6 +3,8 @@ package v1alpha3 import ( + "time" + meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -190,6 +192,10 @@ type ApplierClaim interface { // false when the applier has none it shares this way or the applier claim has not read it. Appliers // sharing one have one operation at a time. SharedSource() (source types.NamespacedName, known bool) + // CatchUpAfter is spec.catchUpAfter, or the kind's default when it is omitted: how long + // the applier may stay behind the branch after a push of GitOps Reverser's own. Zero never + // converges for that reason. + CatchUpAfter() time.Duration // Operation is status.operation, and SetOperation replaces it and status.repairPending. Operation() *ApplyOperation RepairPending() bool @@ -214,6 +220,14 @@ func NewApplierClaim(kind string) ApplierClaim { } } +// durationOr is d, or fallback when it is omitted. +func durationOr(d *metav1.Duration, fallback time.Duration) time.Duration { + if d == nil { + return fallback + } + return d.Duration +} + // defaultedNamespace is namespace, or fallback when it is empty. func defaultedNamespace(namespace, fallback string) string { if namespace == "" { diff --git a/api/v1alpha3/argocdclaim_types.go b/api/v1alpha3/argocdclaim_types.go index f0ad8ea11..eeedf0114 100644 --- a/api/v1alpha3/argocdclaim_types.go +++ b/api/v1alpha3/argocdclaim_types.go @@ -4,6 +4,7 @@ package v1alpha3 import ( "strings" + "time" meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -54,6 +55,27 @@ type ArgoCDClaimSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec.applicationRef is immutable; delete and recreate the ArgoCDClaim to hold another Application" // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.applicationRef.name must not be empty" ApplicationRef meta.NamespacedObjectReference `json:"applicationRef"` + + // Why off by default, where a FluxClaim's is on: a sync runs the Application's hooks, which + // cost more than a no-op apply, and Argo CD's own refresh already compares the live state + // against the tip without a sync. Only the last sync's revision lags. + + // CatchUpAfter is how long the Application stays behind the branch after + // a push of GitOps Reverser's own before a sync of the tip is asked for, + // as a Go duration ("5m"). Such a push changes nothing on the cluster, + // which already holds the edit, but leaves the Application's last sync at + // the commit before it. The interval starts when the Application is first + // seen behind, so a burst of saves costs one sync, hooks included, and one + // at the tip costs none. The sync can still wait for startup, a retry + // backoff or edits being published. "0s" never asks for this reason, and + // drops a sync it asked for that has not started. Otherwise at least "10s" + // and at most "24h". Defaults to "0s". + // +optional + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$" + // +kubebuilder:default="0s" + // +kubebuilder:validation:XValidation:rule="duration(self) == duration('0s') || (duration(self) >= duration('10s') && duration(self) <= duration('24h'))",message="spec.catchUpAfter must be 0s, or a Go duration from 10s to 24h" + CatchUpAfter *metav1.Duration `json:"catchUpAfter,omitempty"` } // ArgoCDClaimStatus is what the applier claim knows about its Application, and @@ -162,7 +184,9 @@ type ArgoCDClaimSource struct { // - after a refused edit, on a GitTarget with spec.onRefusal "Reapply"; // - after a sync reverted an edit made through the API; // - after objects were created through the API in its folders; -// - once at startup, and when a GitTarget first names this ArgoCDClaim. +// - once at startup, and when a GitTarget first names this ArgoCDClaim; +// - once spec.catchUpAfter passed since a push of GitOps Reverser's own; +// - after a CommitRequest with spec.catchUp "AfterPush" pushed a commit. // // GitTargets name it with spec.applierClaimRef. type ArgoCDClaim struct { @@ -195,6 +219,9 @@ func init() { SchemeBuilder.Register(&ArgoCDClaim{}, &ArgoCDClaimList{}) } +// DefaultArgoCDCatchUpAfter is an ArgoCDClaim's spec.catchUpAfter when it is omitted: off. +const DefaultArgoCDCatchUpAfter = 0 + var _ ApplierClaim = (*ArgoCDClaim)(nil) // ClaimKind is ArgoCDClaim. @@ -235,6 +262,11 @@ func (t *ArgoCDClaim) SharedSource() (types.NamespacedName, bool) { return types.NamespacedName{}, false } +// CatchUpAfter is spec.catchUpAfter, or DefaultArgoCDCatchUpAfter when it is omitted. +func (t *ArgoCDClaim) CatchUpAfter() time.Duration { + return durationOr(t.Spec.CatchUpAfter, DefaultArgoCDCatchUpAfter) +} + // Operation is status.operation. func (t *ArgoCDClaim) Operation() *ApplyOperation { return t.Status.Operation } diff --git a/api/v1alpha3/commitrequest_schema_test.go b/api/v1alpha3/commitrequest_schema_test.go new file mode 100644 index 000000000..f6e92dd1c --- /dev/null +++ b/api/v1alpha3/commitrequest_schema_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// commitRequestSchemaHashes is the hash of the generated CommitRequest CRD at each schema revision, +// with the revision line itself left out. A change to the CRD that keeps the revision fails +// TestCommitRequestSchemaRevision: raise CommitRequestSchemaRevision, and the revision line, and add +// the new hash here. +var commitRequestSchemaHashes = map[int]string{ + 1: "ea99cba6116e0f562c0be931c13d6580eabdc2e7ba32d486d20a0a5029163e99", +} + +// commitRequestRevisionLine is the last line of the kind's description. +var commitRequestRevisionLine = regexp.MustCompile( + `Schema revision (\d+), compatible from revision (\d+)\.`) + +// The CommitRequest CRD carries its schema revision as the last line of the kind's description, and +// the revision moves with every change to the generated CRD, so a cluster that installed it on its own +// can be judged by it. +func TestCommitRequestSchemaRevision(t *testing.T) { + raw, err := os.ReadFile("../../config/crd/bases/configbutler.ai_commitrequests.yaml") + require.NoError(t, err) + crd := string(raw) + + match := commitRequestRevisionLine.FindStringSubmatch(crd) + require.NotNil(t, match, "the CRD's description must end with the schema revision line") + revision, _ := strconv.Atoi(match[1]) + floor, _ := strconv.Atoi(match[2]) + require.Equal(t, CommitRequestSchemaRevision, revision, "the revision line and CommitRequestSchemaRevision") + require.Equal(t, CommitRequestSchemaFloor, floor, "the revision line and CommitRequestSchemaFloor") + require.LessOrEqual(t, floor, revision) + + sum := sha256.Sum256([]byte(strings.Replace(crd, match[0], "", 1))) + hash := hex.EncodeToString(sum[:]) + want, recorded := commitRequestSchemaHashes[revision] + require.True(t, recorded, "revision %d has no hash recorded in commitRequestSchemaHashes", revision) + require.Equal(t, want, hash, "the generated CommitRequest CRD changed at revision %d: raise the schema "+ + "revision in api/v1alpha3/commitrequest_types.go, and record the new CRD's hash for it in "+ + "commitRequestSchemaHashes", revision) +} diff --git a/api/v1alpha3/commitrequest_types.go b/api/v1alpha3/commitrequest_types.go index a67819c96..4bfc6b3cd 100644 --- a/api/v1alpha3/commitrequest_types.go +++ b/api/v1alpha3/commitrequest_types.go @@ -5,6 +5,7 @@ package v1alpha3 import ( meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // CommitRequestSpec defines the desired state of CommitRequest. The spec is @@ -22,14 +23,16 @@ import ( // including one that only touches metadata — was rejected as a spec change. // CommitRequestWindow compares its durations as durations, which compares what // the fields mean instead of how they were spelled. -// +kubebuilder:validation:XValidation:rule="self.gitTargetRef == oldSelf.gitTargetRef && has(self.message) == has(oldSelf.message) && (!has(self.message) || self.message == oldSelf.message) && has(self.window) == has(oldSelf.window) && has(self.whenNothingToCommit) == has(oldSelf.whenNothingToCommit) && (!has(self.whenNothingToCommit) || self.whenNothingToCommit == oldSelf.whenNothingToCommit)",message="CommitRequest spec is immutable after creation" +// +kubebuilder:validation:XValidation:rule="self.gitTargetRef == oldSelf.gitTargetRef && has(self.message) == has(oldSelf.message) && (!has(self.message) || self.message == oldSelf.message) && has(self.window) == has(oldSelf.window) && has(self.whenNothingToCommit) == has(oldSelf.whenNothingToCommit) && (!has(self.whenNothingToCommit) || self.whenNothingToCommit == oldSelf.whenNothingToCommit) && has(self.catchUp) == has(oldSelf.catchUp) && (!has(self.catchUp) || self.catchUp == oldSelf.catchUp)",message="CommitRequest spec is immutable after creation" // +kubebuilder:validation:XValidation:rule="!has(self.whenNothingToCommit) || self.whenNothingToCommit != 'CommitEmpty' || has(self.message)",message="whenNothingToCommit: CommitEmpty requires spec.message, because a message is all an empty commit records" type CommitRequestSpec struct { - // GitTargetRef names the GitTarget, in this namespace, whose commit window - // this request attaches to. + // GitTargetRef names the GitTarget whose commit window this request + // attaches to: in this namespace, or namespace names another. The + // GitTarget's spec.allowedCommitRequests must admit this namespace, or the + // request fails with reason "NotAllowed". // +required // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.gitTargetRef.name must not be empty" - GitTargetRef meta.LocalObjectReference `json:"gitTargetRef"` + GitTargetRef meta.NamespacedObjectReference `json:"gitTargetRef"` // Message is the commit message this request asks for. Omitted, the // GitTarget's spec.commit.message.liveTemplate words the commit. @@ -55,9 +58,9 @@ type CommitRequestSpec struct { // for one, and the timers that close the window once it has. They replace the // GitTarget's timers for that window. A request never opens a window: only a // write does. Omitted, the request attaches to the author's current or next - // window within 2s and closes it 2s later. + // window within 5s and closes it 2s later. // +optional - // +kubebuilder:default={attach: "CurrentOrNext", attachTimeout: "2s", maxDuration: "2s"} + // +kubebuilder:default={attach: "CurrentOrNext", attachTimeout: "5s", maxDuration: "2s"} Window *CommitRequestWindow `json:"window,omitempty"` // WhenNothingToCommit decides what happens when the request ends with nothing @@ -71,8 +74,32 @@ type CommitRequestSpec struct { // +kubebuilder:validation:Enum=Resolve;CommitEmpty // +kubebuilder:default=Resolve WhenNothingToCommit NothingToCommitAction `json:"whenNothingToCommit,omitempty"` + + // CatchUp decides when the GitTarget's applier is asked to apply the branch + // after this request pushes a commit. "AsDefinedOnClaim" leaves it to the + // applier claim's spec.catchUpAfter. "AfterPush" asks right after the push. + // The apply takes the latest branch tip, which may hold later commits, and + // saves pushed close together can share one apply; the Applied condition + // reports how it went. The cluster already holds the saved objects: + // the apply brings the applier's own state, such as a Kustomization's + // status.lastAppliedRevision, to the tip. Defaults to "AsDefinedOnClaim". + // +optional + // +kubebuilder:validation:Enum=AsDefinedOnClaim;AfterPush + // +kubebuilder:default=AsDefinedOnClaim + CatchUp CatchUpPolicy `json:"catchUp,omitempty"` } +// CatchUpPolicy is when a CommitRequest's commit is applied by the GitTarget's +// applier. +type CatchUpPolicy string + +const ( + // CatchUpAsDefinedOnClaim leaves the apply to the applier claim's spec.catchUpAfter. + CatchUpAsDefinedOnClaim CatchUpPolicy = "AsDefinedOnClaim" + // CatchUpAfterPush asks the applier to apply the tip right after the commit is pushed. + CatchUpAfterPush CatchUpPolicy = "AfterPush" +) + // NothingToCommitAction is what a CommitRequest does when it ends with nothing // to commit. type NothingToCommitAction string @@ -119,14 +146,14 @@ type CommitRequestWindow struct { Attach AttachPolicy `json:"attach,omitempty"` // AttachTimeout is how long the request waits for a window, as a Go duration, - // from when GitOps Reverser picks the request up. "0s" attaches only to a - // window already open. A request whose wait ran out never takes a later - // window. At most "5m". Defaults to "2s", which covers a write's wait for its - // audit event. + // from when GitOps Reverser picks the request up, which is after its own + // author was found. "0s" attaches only to a window already open. A request + // whose wait ran out never takes a later window. At most "5m". Defaults to + // "5s", which covers a write's wait for its audit event. // +optional // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$" - // +kubebuilder:default="2s" + // +kubebuilder:default="5s" // +kubebuilder:validation:XValidation:rule="duration(self) <= duration('5m')",message="spec.window.attachTimeout must not exceed 5m" AttachTimeout *metav1.Duration `json:"attachTimeout,omitempty"` @@ -156,30 +183,52 @@ type CommitRequestWindow struct { // // - Ready (summary): True once the request reached a terminal outcome that // is not an error — a pushed commit, or a benign no-commit (nothing to -// save, already present, or a foreign open window). False while in -// progress or when it failed. +// save, already present, or a foreign open window). Unknown while in +// progress, from the first status write, which records the author. False +// when it failed. // - Reconciling / Stalled: the kstatus progress / blocked pair. // Reconciling=True while finalizing; Stalled=True when the finalize failed // and needs attention. -// - AuthorAttributed (domain): binary and settled immediately. True -// (AttributedFromAdmission) when the submitter captured at admission named -// the commit author; False (CommitterFallback) when capture ran but no -// admission record exists, or False (AuthorCaptureDisabled) when capture -// is disabled. In either False case the request claims no actor. False is -// not a failure and does not affect Ready; the final Git author remains -// the matching watch window's author. // - Pushed (domain): True once the commit is in the remote repository. +// - Applied (domain): with spec.catchUp "AfterPush", whether the applier +// caught up with the commit. It never affects Ready. type CommitRequestStatus struct { // ObservedGeneration is the latest generation observed by the controller. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // Conditions are the latest observations of this CommitRequest's state. Ready, - // Reconciling and Stalled follow kstatus: Ready is True once the request ended - // without error, with a pushed commit or with nothing to save. Pushed is True - // once the commit is on the remote. AuthorAttributed says whether the - // request's creator, recorded at admission, names the commit author; False is - // not a failure and does not affect Ready. + // Author is who created this request, as the audit events of the cluster + // it was created on name them. Empty when they name nobody, or when author + // attribution is off: then the request saves only writes that name nobody + // either. + // +optional + Author string `json:"author,omitempty"` + + // Conditions are the latest observations of this CommitRequest's state. + // Reconciling and Stalled follow kstatus. + // + // Three conditions say how far the save got: + // + // - Ready (kstatus): Unknown while the save runs. True once the request + // ended without error, with a pushed commit or with nothing to save; it + // never waits for an apply. False when it failed: reason NotAllowed, + // GitTargetNotFound, SaveInProgress or FinalizeFailed. + // - Pushed: True once the commit is on the remote. + // - Applied: set only with spec.catchUp "AfterPush", once Ready is True. + // + // Applied says whether the cluster's applier caught up with the commit: + // + // - Unknown, reason Applying or Retrying: the apply runs, or did not + // complete and is asked again; the message says why. + // - True, reason Applied: the applier applied a revision that holds the + // commit, named in status.appliedRevision. + // - True, reason NothingToApply, NoApplierClaim or ApplierGone: no apply + // was needed. The request pushed no commit, or the GitTarget names no + // applier claim, or stopped naming the one asked; the cluster already + // holds the saved objects. + // - False, reason CommitDropped: the branch no longer holds the commit; + // its history was rewritten after the push. Only this is False. + // - Unknown, reason Lost: GitOps Reverser restarted before it knew. // +optional // +listType=map // +listMapKey=type @@ -187,6 +236,12 @@ type CommitRequestStatus struct { // +patchMergeKey=type Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + // GitTargetUID is the UID of the GitTarget whose spec.allowedCommitRequests + // admitted this request. A GitTarget with another UID, such as one created + // after the request under the same name, must admit it again. + // +optional + GitTargetUID string `json:"gitTargetUID,omitempty"` + // Branch is the branch the commit is made on. Set once the request ends. // +optional Branch string `json:"branch,omitempty"` @@ -194,24 +249,44 @@ type CommitRequestStatus struct { // Commit is the commit hash. Set once the commit is pushed (Pushed=True). // +optional Commit string `json:"commit,omitempty"` + + // AppliedRevision is the revision the GitTarget's applier reported applying + // for this request, as in a Kustomization's status.lastAppliedRevision. It + // holds the request's commit, and may be a later one. Set once Applied is + // True with reason Applied. + // +optional + AppliedRevision string `json:"appliedRevision,omitempty"` } +// The last line of the kind's description is its schema revision, which a cluster that installed the +// CRD on its own, as a source cluster does, can be judged by. It ships in the CRD, so the same +// published schema reads it back: CommitRequestSchemaRevision and CommitRequestSchemaFloor below +// must say the same, and TestCommitRequestSchemaRevision fails when the generated CRD changes and +// they do not move. Raise the revision with every change to the generated CRD, and the floor ("compatible +// from") with a change that breaks an older reader or writer: a new required field, a removed field, +// or a changed meaning. + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="GitTarget",type=string,JSONPath=`.spec.gitTargetRef.name` // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` // +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +// +kubebuilder:printcolumn:name="Author",type=string,JSONPath=`.status.author` // +kubebuilder:printcolumn:name="Commit",type=string,JSONPath=`.status.commit` -// +kubebuilder:printcolumn:name="AuthorAttributed",type=string,JSONPath=`.status.conditions[?(@.type=="AuthorAttributed")].status`,priority=1 // +kubebuilder:printcolumn:name="Pushed",type=string,JSONPath=`.status.conditions[?(@.type=="Pushed")].status`,priority=1 +// +kubebuilder:printcolumn:name="Applied",type=string,JSONPath=`.status.conditions[?(@.type=="Applied")].status`,priority=1 // +kubebuilder:printcolumn:name="Branch",type=string,JSONPath=`.status.branch`,priority=1 // +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // CommitRequest asks for a save: it attaches to its author's current or next // commit window on a GitTarget, closes that window on its own timers, and -// gives the commit its message. Its spec is immutable, and status.commit -// reports the commit once it is on the remote. +// gives the commit its message. Create it on the cluster GitOps Reverser runs +// in, or on a source cluster whose KubeProvider sets spec.commitRequests. Its +// spec is immutable, and status.commit reports the commit once it is on the +// remote. +// +// Schema revision 1, compatible from revision 1. type CommitRequest struct { metav1.TypeMeta `json:",inline"` @@ -239,6 +314,24 @@ type CommitRequestList struct { Items []CommitRequest `json:"items"` } +// GitTargetKey is the GitTarget spec.gitTargetRef names: in the request's namespace unless the ref +// names another. +func (s *CommitRequest) GitTargetKey() types.NamespacedName { + namespace := s.Spec.GitTargetRef.Namespace + if namespace == "" { + namespace = s.Namespace + } + return types.NamespacedName{Namespace: namespace, Name: s.Spec.GitTargetRef.Name} +} + +// CommitRequestSchemaRevision and CommitRequestSchemaFloor are the revision of the CommitRequest +// schema this operator was built with, and the oldest revision it can serve: the last line of the +// kind's description. +const ( + CommitRequestSchemaRevision = 1 + CommitRequestSchemaFloor = 1 +) + func init() { SchemeBuilder.Register(&CommitRequest{}, &CommitRequestList{}) } diff --git a/api/v1alpha3/fluxclaim_types.go b/api/v1alpha3/fluxclaim_types.go index b4cb8e1ca..e50554f2a 100644 --- a/api/v1alpha3/fluxclaim_types.go +++ b/api/v1alpha3/fluxclaim_types.go @@ -3,6 +3,8 @@ package v1alpha3 import ( + "time" + meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -45,6 +47,29 @@ type FluxClaimSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec.kustomizationRef is immutable; delete and recreate the FluxClaim to hold another Kustomization" // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.kustomizationRef.name must not be empty" KustomizationRef meta.NamespacedObjectReference `json:"kustomizationRef"` + + // Why a timer: a push of GitOps Reverser's own needs no apply for the cluster (the flush + // already put the edit there), so none is asked for at once; but what reads the applier's own + // state (its applied revision, its source artifact) would stay at the last foreign commit until + // someone else pushed. One apply per interval keeps that level and batches a burst of saves. + + // CatchUpAfter is how long the Kustomization stays behind the branch after + // a push of GitOps Reverser's own before an apply of the tip is asked for, + // as a Go duration ("1m", "90s"). Such a push changes nothing on the + // cluster, which already holds the edit, but leaves + // status.lastAppliedRevision and the GitRepository's artifact at the commit + // before it. The interval starts when the Kustomization is first seen + // behind, so a burst of saves costs one apply, and one at the tip costs + // none. The apply can still wait for startup, a retry backoff or edits + // being published. "0s" never asks for this reason, and drops an apply it + // asked for that has not started. Otherwise at least "10s" and at most + // "24h". Defaults to "1m". + // +optional + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$" + // +kubebuilder:default="1m" + // +kubebuilder:validation:XValidation:rule="duration(self) == duration('0s') || (duration(self) >= duration('10s') && duration(self) <= duration('24h'))",message="spec.catchUpAfter must be 0s, or a Go duration from 10s to 24h" + CatchUpAfter *metav1.Duration `json:"catchUpAfter,omitempty"` } // FluxClaimStatus is what the applier claim knows about its Kustomization, and @@ -165,7 +190,9 @@ type FluxClaimSource struct { // - after a refused edit, on a GitTarget with spec.onRefusal "Reapply"; // - after an apply reverted an edit made through the API; // - after objects were created through the API in its folders; -// - once at startup, and when a GitTarget first names this FluxClaim. +// - once at startup, and when a GitTarget first names this FluxClaim; +// - once spec.catchUpAfter passed since a push of GitOps Reverser's own; +// - after a CommitRequest with spec.catchUp "AfterPush" pushed a commit. // // GitTargets name it with spec.applierClaimRef. type FluxClaim struct { @@ -198,6 +225,9 @@ func init() { SchemeBuilder.Register(&FluxClaim{}, &FluxClaimList{}) } +// DefaultFluxCatchUpAfter is a FluxClaim's spec.catchUpAfter when it is omitted. +const DefaultFluxCatchUpAfter = time.Minute + var _ ApplierClaim = (*FluxClaim)(nil) // ClaimKind is FluxClaim. @@ -236,6 +266,11 @@ func (t *FluxClaim) SharedSource() (types.NamespacedName, bool) { return types.NamespacedName{Namespace: source.Namespace, Name: source.Name}, true } +// CatchUpAfter is spec.catchUpAfter, or DefaultFluxCatchUpAfter when it is omitted. +func (t *FluxClaim) CatchUpAfter() time.Duration { + return durationOr(t.Spec.CatchUpAfter, DefaultFluxCatchUpAfter) +} + // Operation is status.operation. func (t *FluxClaim) Operation() *ApplyOperation { return t.Status.Operation } diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index 6d0c41ada..6f9d8f116 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -184,6 +184,29 @@ type GitTargetSpec struct { // +optional Commit *GitTargetCommitSpec `json:"commit,omitempty"` + // AllowedCommitRequests says which CommitRequests may save this GitTarget's + // commit windows. Each grant sets one of: + // + // - consumerNamespaces: namespaces of the cluster GitOps Reverser runs + // in. They admit requests only when spec.kubeProviderRef names an + // inCluster KubeProvider. + // - sourceNamespaces: namespaces of the cluster spec.kubeProviderRef + // names, which must set spec.commitRequests. They admit requests only + // when that KubeProvider names a kubeconfig. + // + // Omitted, the CommitRequests in this GitTarget's namespace are admitted, + // as one grant with consumerNamespaces from: "Same". An empty list ([]) + // admits none. A list replaces the default. A request no grant admits + // fails with reason "NotAllowed". + // + // An admitted request saves its creator's whole commit window on this + // GitTarget, writes outside the request's namespace included. + // +optional + // +listType=atomic + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:default={{consumerNamespaces: {from: Same}}} + AllowedCommitRequests []GitTargetCommitRequestGrant `json:"allowedCommitRequests"` + // The scan keeps running while writes are suspended: freezing status.placement too would leave // a stale answer at the moment it costs most. A suspended target is Ready=True reason // Suspended, because not writing is the configured outcome rather than ill health. A local diff --git a/api/v1alpha3/kubeprovider_types.go b/api/v1alpha3/kubeprovider_types.go index fe31a03eb..ab5c61727 100644 --- a/api/v1alpha3/kubeprovider_types.go +++ b/api/v1alpha3/kubeprovider_types.go @@ -28,6 +28,10 @@ import ( // mirroring through it, so the credential is immutable, including switching // arms. A rotated kubeconfig is a change to the Secret, which stays allowed. // +kubebuilder:validation:XValidation:rule="has(self.inCluster) == has(oldSelf.inCluster) && (!has(self.secretRef) || self.secretRef == oldSelf.secretRef)",message="spec.secretRef and spec.inCluster are immutable; delete and recreate the KubeProvider to point it at a different cluster" +// +// The cluster GitOps Reverser runs in has no switch: its CommitRequests are +// always served, by the operator itself. +// +kubebuilder:validation:XValidation:rule="!has(self.inCluster) || !has(self.commitRequests) || self.commitRequests == 'Ignore'",message="spec.commitRequests must be Ignore beside spec.inCluster: the CommitRequests of the cluster GitOps Reverser runs in are always served" type KubeProviderSpec struct { // SecretRef names the Secret, in this namespace, that holds a kubeconfig for // the cluster. With no key, the kubeconfig is read from "value", then @@ -99,8 +103,46 @@ type KubeProviderSpec struct { // audit events. // +optional Attribution *KubeProviderAttribution `json:"attribution,omitempty"` + + // CommitRequests says whether GitOps Reverser serves the CommitRequests + // created on this cluster, for the GitTargets that mirror through this + // KubeProvider: + // + // - "Ignore": they are left alone. + // - "Reconcile": the requests in the namespaces this KubeProvider lends + // its GitTargets in spec.allowedGitTargets are served, and their status + // is written. A finished request stays. + // - "ReconcileAndDeleteFinished": as "Reconcile", and a finished request + // is annotated configbutler.ai/delete-after and deleted once that time + // has passed. + // + // Defaults to "Ignore". Only "Ignore" goes with inCluster: the + // CommitRequests of the cluster GitOps Reverser runs in are always served. + // + // The credential needs get, list and watch on commitrequests, and update + // on commitrequests/status, in the configbutler.ai group, in each lent + // namespace; "ReconcileAndDeleteFinished" also needs patch and delete on + // commitrequests. The CommitRequest CRD must be installed on the cluster. + // The CommitRequestsServed condition reports how it goes. + // +optional + // +kubebuilder:validation:Enum=Ignore;Reconcile;ReconcileAndDeleteFinished + // +kubebuilder:default=Ignore + CommitRequests CommitRequestsPolicy `json:"commitRequests,omitempty"` } +// CommitRequestsPolicy is whether a KubeProvider's CommitRequests are served, +// and whether finished ones are deleted. +type CommitRequestsPolicy string + +const ( + // CommitRequestsIgnore leaves the cluster's CommitRequests alone. + CommitRequestsIgnore CommitRequestsPolicy = "Ignore" + // CommitRequestsReconcile serves the cluster's CommitRequests and keeps finished ones. + CommitRequestsReconcile CommitRequestsPolicy = "Reconcile" + // CommitRequestsReconcileAndDeleteFinished serves them and deletes finished ones. + CommitRequestsReconcileAndDeleteFinished CommitRequestsPolicy = "ReconcileAndDeleteFinished" +) + // KubeProviderInCluster selects the operator's own identity. It has no fields: // the arm being set is the whole statement. type KubeProviderInCluster struct{} @@ -169,6 +211,11 @@ type KubeProviderStatus struct { // +optional AuditRoute string `json:"auditRoute,omitempty"` + // CommitRequests is what this KubeProvider serves of its cluster's + // CommitRequests. Absent while spec.commitRequests is "Ignore". + // +optional + CommitRequests *KubeProviderCommitRequestsStatus `json:"commitRequests,omitempty"` + // Conditions are the latest observations of this KubeProvider's state. Ready, // Reconciling and Stalled follow kstatus; Stalled=True with reason // InClusterNotAuthorized or AuditRouteConflict means the KubeProvider is not @@ -182,6 +229,19 @@ type KubeProviderStatus struct { // NoAuditDelivery (no audit events arrive at all), or RouteUnused (audit // events arrive, but never on this route). It is absent when author // attribution is off. + // + // CommitRequestsServed, which is not part of Ready either, says whether + // this cluster's CommitRequests are served. It is absent while + // spec.commitRequests is "Ignore". Each reason that is not True ends with + // its fix: + // + // - True, Watching: watched in status.commitRequests.namespaces. + // - True, NothingToWatch: no GitTarget mirroring through this KubeProvider + // is lent a namespace. + // - False, CRDNotInstalled: the cluster serves no CommitRequest CRD. + // - False, VersionNotServed: the CRD does not serve v1alpha3. + // - False, Forbidden: the credential may not list or watch them. + // - Unknown, Unreachable: the cluster's API could not be read. // +optional // +listType=map // +listMapKey=type @@ -190,6 +250,17 @@ type KubeProviderStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } +// KubeProviderCommitRequestsStatus is what a KubeProvider serves of its +// cluster's CommitRequests. +type KubeProviderCommitRequestsStatus struct { + // Namespaces are the namespaces whose CommitRequests are watched: the ones + // this KubeProvider lends the GitTargets that mirror through it. "*" is + // every namespace, in one watch. + // +optional + // +listType=set + Namespaces []string `json:"namespaces,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` @@ -197,6 +268,7 @@ type KubeProviderStatus struct { // +kubebuilder:printcolumn:name="FactsReceived",type=string,JSONPath=`.status.conditions[?(@.type=="AuditFactsReceived")].status` // +kubebuilder:printcolumn:name="Route",type=string,JSONPath=`.status.auditRoute`,priority=1 // +kubebuilder:printcolumn:name="Validated",type=string,JSONPath=`.status.conditions[?(@.type=="Validated")].status`,priority=1 +// +kubebuilder:printcolumn:name="Saves",type=string,JSONPath=`.status.conditions[?(@.type=="CommitRequestsServed")].reason`,priority=1 // +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` @@ -237,6 +309,20 @@ func (p *KubeProvider) Key() types.NamespacedName { return types.NamespacedName{Namespace: p.Namespace, Name: p.Name} } +// ServesCommitRequests reports whether GitOps Reverser serves the CommitRequests created on this +// provider's cluster, through its own watch of it: spec.commitRequests is one of the two Reconcile +// values, and the provider names a kubeconfig. +func (p *KubeProvider) ServesCommitRequests() bool { + return !p.IsInCluster() && (p.Spec.CommitRequests == CommitRequestsReconcile || + p.Spec.CommitRequests == CommitRequestsReconcileAndDeleteFinished) +} + +// DeletesFinishedCommitRequests reports whether finished CommitRequests on this provider's cluster +// are deleted. +func (p *KubeProvider) DeletesFinishedCommitRequests() bool { + return p.ServesCommitRequests() && p.Spec.CommitRequests == CommitRequestsReconcileAndDeleteFinished +} + // IsInCluster reports whether this provider asks for the operator's own identity (spec.inCluster). // Whether it may have it is the installation's decision, not this object's. func (p *KubeProvider) IsInCluster() bool { diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index 62900075b..e666b9eef 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -88,7 +88,7 @@ func (in *ArgoCDClaim) DeepCopyInto(out *ArgoCDClaim) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -162,6 +162,11 @@ func (in *ArgoCDClaimSpec) DeepCopyInto(out *ArgoCDClaimSpec) { *out = *in out.KubeProviderRef = in.KubeProviderRef out.ApplicationRef = in.ApplicationRef + if in.CatchUpAfter != nil { + in, out := &in.CatchUpAfter, &out.CatchUpAfter + *out = new(v1.Duration) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDClaimSpec. @@ -629,7 +634,7 @@ func (in *FluxClaim) DeepCopyInto(out *FluxClaim) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -709,6 +714,11 @@ func (in *FluxClaimSpec) DeepCopyInto(out *FluxClaimSpec) { *out = *in out.KubeProviderRef = in.KubeProviderRef out.KustomizationRef = in.KustomizationRef + if in.CatchUpAfter != nil { + in, out := &in.CatchUpAfter, &out.CatchUpAfter + *out = new(v1.Duration) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluxClaimSpec. @@ -1188,6 +1198,31 @@ func (in *GitTarget) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitTargetCommitRequestGrant) DeepCopyInto(out *GitTargetCommitRequestGrant) { + *out = *in + if in.ConsumerNamespaces != nil { + in, out := &in.ConsumerNamespaces, &out.ConsumerNamespaces + *out = new(ConsumerNamespaces) + (*in).DeepCopyInto(*out) + } + if in.SourceNamespaces != nil { + in, out := &in.SourceNamespaces, &out.SourceNamespaces + *out = make([]SourceNamespace, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetCommitRequestGrant. +func (in *GitTargetCommitRequestGrant) DeepCopy() *GitTargetCommitRequestGrant { + if in == nil { + return nil + } + out := new(GitTargetCommitRequestGrant) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitTargetCommitSpec) DeepCopyInto(out *GitTargetCommitSpec) { *out = *in @@ -1345,6 +1380,13 @@ func (in *GitTargetSpec) DeepCopyInto(out *GitTargetSpec) { *out = new(GitTargetCommitSpec) (*in).DeepCopyInto(*out) } + if in.AllowedCommitRequests != nil { + in, out := &in.AllowedCommitRequests, &out.AllowedCommitRequests + *out = make([]GitTargetCommitRequestGrant, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetSpec. @@ -1497,6 +1539,26 @@ func (in *KubeProviderClient) DeepCopy() *KubeProviderClient { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeProviderCommitRequestsStatus) DeepCopyInto(out *KubeProviderCommitRequestsStatus) { + *out = *in + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProviderCommitRequestsStatus. +func (in *KubeProviderCommitRequestsStatus) DeepCopy() *KubeProviderCommitRequestsStatus { + if in == nil { + return nil + } + out := new(KubeProviderCommitRequestsStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeProviderGitTargetGrant) DeepCopyInto(out *KubeProviderGitTargetGrant) { *out = *in @@ -1622,6 +1684,11 @@ func (in *KubeProviderSpec) DeepCopy() *KubeProviderSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeProviderStatus) DeepCopyInto(out *KubeProviderStatus) { *out = *in + if in.CommitRequests != nil { + in, out := &in.CommitRequests, &out.CommitRequests + *out = new(KubeProviderCommitRequestsStatus) + (*in).DeepCopyInto(*out) + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) diff --git a/charts/gitops-reverser/README.md b/charts/gitops-reverser/README.md index c9c610cea..24da08aea 100644 --- a/charts/gitops-reverser/README.md +++ b/charts/gitops-reverser/README.md @@ -27,14 +27,14 @@ For the chart's optional starter `quickstart` block, see [`docs/configuration.md ## Prerequisites - Kubernetes 1.36 (probably works for older versions, but not tested) -- cert-manager (for TLS certificates) - **Valkey or Redis: optional but advised.** The default install runs without it in `configured-author` mode (watches cold-replay on restart). Add it to unlock warm-restart resume - cursors, CommitRequest author capture (the admission webhook), and attributed-author mode; enabling - any of those requires a non-empty `queue.redis.addr`. -- **Optional, attributed-author mode only:** kube-apiserver audit delivery, which adds - mirrored-resource commit-author attribution. Without it the operator still mirrors state, authored by - the configured committer. + cursors and attributed-author mode; enabling either requires a non-empty `queue.redis.addr`. +- **Optional, attributed-author mode only:** kube-apiserver audit delivery, which adds commit-author + attribution for mirrored resources and `CommitRequest`s, and cert-manager, which mints the audit + ingress certificates (or bring the Secrets yourself with `servers.audit.tls.certManager: false`). + Without them the operator still mirrors state, authored by the configured committer. The default + install renders no certificate, so it needs no cert-manager. ## Installation @@ -198,7 +198,7 @@ nodeSelector: | `quickstart.gitBranch.branch` | Branch the starter `GitBranch` writes. Must match `quickstart.gitProvider.branchPatterns` | `main` | | `quickstart.gitTarget.path` | Repository path used by the starter `GitTarget`; set `.` only to deliberately target the repo root | `live-cluster` | | `quickstart.watchRule.rules` | Rules used by the starter `WatchRule` | `configmaps create/update/delete` | -| `queue.redis.addr` | Redis/Valkey endpoint (`host:port`). Optional but advised: empty means cold-replay on restart. Set it for warm-restart cursors and for the admission webhook to actually record CommitRequest authors (admission runs as a no-op without it); **required** when `attribution.enabled=true` unless `attribution.transport=memory` | `""` | +| `queue.redis.addr` | Redis/Valkey endpoint (`host:port`). Optional but advised: empty means cold-replay on restart. Set it for warm-restart cursors; **required** when `attribution.enabled=true` unless `attribution.transport=memory` | `""` | | `queue.redis.auth.existingSecret` | Name of a pre-created Secret holding the Redis password (only used when `queue.redis.addr` is set) | `""` | | `queue.redis.auth.existingSecretKey` | Key within the Secret that holds the password | `password` | | `queue.redis.auth.username` | Optional Redis ACL username | `""` | @@ -220,7 +220,6 @@ nodeSelector: | `kubeProvider.secretRef.key` | Key within that Secret. Empty reads `value` then `value.yaml` | `""` | | `kubeProvider.allowedGitTargets` | Rendered as the provider's `spec.allowedGitTargets`: which namespaces' `GitTarget`s may mirror through it, and what. Each grant has a `consumerNamespaces` block (`from`: `Same`, `Names`, `Selector`, `All`), and `sourceNamespaces`, `clusterResources` or both beside it. **Unset, the field is omitted and the CRD defaults it to the release namespace mirroring its own**; a list is the whole policy, and `[]` admits nobody, the release namespace included: a refused `GitTarget` names `spec.allowedGitTargets`. `quickstart.enabled` adds a grant for the quickstart namespace, and [`values-playground.yaml`](values-playground.yaml) grants every namespace its own. With an `inCluster` provider a grant bypasses live namespace RBAC through the operator's identity | `null` | | `kubeProvider.allowedApplierClaims` | Rendered as the provider's `spec.allowedApplierClaims`: which namespaces' `FluxClaim`s and `ArgoCDClaim`s may hold an applier through it (`consumerNamespaces`), and in which `applierNamespaces` it may live, such as `flux-system`. Admission to mirror is no admission to apply. Unset, the CRD defaults it to the release namespace holding appliers in its own; `[]` admits nobody | `null` | -| `servers.admission.enabled` | Install the validate-operator-types admission webhook that captures CommitRequest authors (a form of author attribution). Enabled by default; a no-op until `queue.redis.addr` is set | `true` | | `rbac.create` | Create the manager ClusterRole and its binding | `true` | | `rbac.watchTypes.mode` | Which types a `WatchRule` may read. `any` grants cluster-wide read on everything, including every Secret in the cluster. `selected` grants read on `rbac.watchTypes.selected` only, so the reverser cannot list or watch Secrets (it keeps `get` on named Secrets it is pointed at). See [`docs/rbac.md`](../../docs/rbac.md) | `any` | | `rbac.watchTypes.selected` | Types to grant when `mode: selected`, as `{apiGroups, resources}` entries (verbs are always `get,list,watch`). Required and non-empty in that mode; `namespaces`, `customresourcedefinitions` and `apiservices` come from the manager role and must not be restated | `[]` | @@ -233,13 +232,9 @@ nodeSelector: | `service.ports.receiver` | Port of the `-receiver` Service when `servers.receiver.enabled=true` | `80` | | `service.ports.metrics` | Service port for metrics | `8080` | | `servers.healthProbe.bindAddress` | Liveness/readiness probe bind address (`--health-probe-bind-address`) | `:8081` | -| `servers.admission.port` | Admission webhook container/Service port | `9443` | -| `servers.admission.timeoutSeconds` | Admission webhook timeout (failurePolicy is Ignore) | `2` | -| `servers.admission.tls.certManager` | Mint the admission serving cert via cert-manager (false = BYO via `secretNameOverride`) | `true` | -| `servers.admission.tls.secretNameOverride` | Override Secret name for the admission serving cert | `-admission-server-cert` | | `certManager.enabled` | Use cert-manager to mint the chart's serving/client certs | `true` | | `certManager.issuer.name` | Name of the (shared) cert-manager issuer | `selfsigned-issuer` | -| `certManager.issuer.create` | Create the self-signed issuer (set false to reuse an existing one) | `true` | +| `certManager.issuer.create` | Create the self-signed issuer when a certificate needs it (set false to reuse an existing one) | `true` | | `servers.metrics.tls.certManager` | Mint the metrics serving cert via cert-manager (used when `servers.metrics.tls.enabled`) | `true` | | `servers.audit.tls.certManager` | Let the chart mint audit TLS Secrets (server + kube-apiserver client) via cert-manager when `attribution.enabled=true` | `true` | | `servers.audit.tls.rootCA.secretNameOverride` | Override Secret name for the audit root CA | `-audit-root-ca` | diff --git a/charts/gitops-reverser/templates/_helpers.tpl b/charts/gitops-reverser/templates/_helpers.tpl index 103ad0fb8..d4a7e1171 100644 --- a/charts/gitops-reverser/templates/_helpers.tpl +++ b/charts/gitops-reverser/templates/_helpers.tpl @@ -93,20 +93,6 @@ Default Secret name for the kube-apiserver audit client certificate. {{- .Values.servers.audit.tls.client.secretNameOverride | default (printf "%s-audit-client-cert" (include "gitops-reverser.fullname" .)) -}} {{- end }} -{{/* -Default Secret name for the validating admission (validate-operator-types) serving certificate. -*/}} -{{- define "gitops-reverser.admissionServerSecretName" -}} -{{- .Values.servers.admission.tls.secretNameOverride | default (printf "%s-admission-server-cert" (include "gitops-reverser.fullname" .)) -}} -{{- end }} - -{{/* -Name of the cert-manager Certificate that mints the admission serving cert. Used both -as the Certificate name and in the webhook's cert-manager.io/inject-ca-from annotation. -*/}} -{{- define "gitops-reverser.admissionServerCertName" -}} -{{- printf "%s-admission-server-cert" (include "gitops-reverser.fullname" .) -}} -{{- end }} {{/* Name of the push receiver Service. @@ -137,3 +123,21 @@ The audit route of the KubeProvider the chart renders: . 1 for the whole chart, so there is no configuration this could catch. - -The admission webhook is deliberately NOT gated here: it stays enabled without Redis and simply -no-ops command-author capture. */ -}} {{- if and .Values.attribution.enabled (eq .Values.attribution.transport "redis") (eq (trim .Values.queue.redis.addr) "") -}} {{- fail "attribution.enabled=true with attribution.transport=redis requires queue.redis.addr: the redis transport carries audit facts on Redis streams. Set queue.redis.addr, select attribution.transport=memory to run attribution in-process on a single replica, or leave attribution.enabled=false." -}} diff --git a/charts/gitops-reverser/values.schema.json b/charts/gitops-reverser/values.schema.json index 4eb69f9d7..2f8bf2fee 100644 --- a/charts/gitops-reverser/values.schema.json +++ b/charts/gitops-reverser/values.schema.json @@ -221,32 +221,6 @@ } } }, - "admission": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { "type": "boolean" }, - "bindAddress": { "type": "string" }, - "port": { "$ref": "#/$defs/port" }, - "timeoutSeconds": { - "type": "integer", - "minimum": 1, - "maximum": 30, - "description": "The apiserver rejects a webhook timeout outside 1-30s." - }, - "tls": { - "type": "object", - "additionalProperties": false, - "properties": { - "certManager": { "type": "boolean" }, - "certPath": { "type": "string" }, - "certName": { "type": "string" }, - "certKey": { "type": "string" }, - "secretNameOverride": { "$ref": "#/$defs/secretNameOverride" } - } - } - } - }, "audit": { "type": "object", "additionalProperties": false, diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index e255e8905..748090355 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -139,6 +139,9 @@ applierAccess: # backs them all, so the issuer is genuinely cross-cutting and lives here; each server # opts into a cert via servers..tls.certManager. (Deliberately NOT duplicated per # server — that one shared resource is the exception to the "servers as peers" rule.) +# The issuer renders only when some certificate does: the audit ingress certificates +# with attribution.enabled, or the metrics certificate with servers.metrics.tls.enabled. +# The default install renders neither, so it needs no cert-manager. certManager: enabled: true issuer: @@ -174,36 +177,6 @@ servers: # Leave empty to use the chart-generated Secret name. secretNameOverride: "" - # Validating admission server hosting the validate-operator-types webhook, which captures - # the submitter of each CommitRequest at admission and records it as the request's named actor - # (read back by the controller with no wait). Independent of `attribution` below: - # attribution governs *mirrored-resource* authorship (from audit), this governs - # *command* authorship (from admission). Off → CommitRequests claim no actor and report - # AuthorAttributed=False. See - # docs/spec/commitrequest-admission-authorship.md. - # - # Command-author capture is Redis-backed: the captured author is written to Redis at admission and - # read back at reconcile. Enabled by default so the webhook is always installed, but it is a no-op - # without Redis: the pod runs healthy and CommitRequests claim no actor - # (AuthorAttributed=False) until queue.redis.addr is set. - admission: - enabled: true - bindAddress: ":9443" - port: 9443 - # Bounded per-request wait before the apiserver gives up on the webhook. - # failurePolicy is Ignore, so a slow/unreachable backend admits the command anyway. - timeoutSeconds: 2 - tls: - # The admission server always serves TLS (the apiserver requires HTTPS); the only - # choice is who mints the cert. certManager=false means BYO via secretNameOverride - # (and patch the webhook caBundle out of band). - certManager: true - certPath: "/tmp/k8s-webhook-server/serving-certs" - certName: "tls.crt" - certKey: "tls.key" - # Leave empty to use the chart-generated Secret name. - secretNameOverride: "" - audit: # host:port the audit ingress server binds to (passed as --audit-bind-address). bindAddress: "0.0.0.0:9444" diff --git a/cmd/main.go b/cmd/main.go index 0395a8f18..7da0cc033 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -34,8 +34,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" - ctrladmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/applier" @@ -58,15 +56,14 @@ var ( ) const ( - flagParseFailureExitCode = 2 - defaultAdmissionWebhookBindAddr = ":9443" - defaultAuditBindAddr = "0.0.0.0:9444" - defaultAuditMaxBodyBytes = int64(10 * 1024 * 1024) - defaultAuditReadTimeout = 15 * time.Second - defaultAuditWriteTimeout = 30 * time.Second - defaultAuditIdleTimeout = 60 * time.Second - defaultAuditShutdownTimeout = 10 * time.Second - defaultBranchBufferMaxSizeStr = "8Mi" + flagParseFailureExitCode = 2 + defaultAuditBindAddr = "0.0.0.0:9444" + defaultAuditMaxBodyBytes = int64(10 * 1024 * 1024) + defaultAuditReadTimeout = 15 * time.Second + defaultAuditWriteTimeout = 30 * time.Second + defaultAuditIdleTimeout = 60 * time.Second + defaultAuditShutdownTimeout = 10 * time.Second + defaultBranchBufferMaxSizeStr = "8Mi" ) func init() { @@ -92,9 +89,7 @@ func main() { "metricsAddr", cfg.metricsAddr, "metricsInsecure", cfg.metricsInsecure, "auditBindAddress", cfg.auditBindAddress, - "auditInsecure", cfg.auditInsecure, - "admissionWebhookEnabled", cfg.admissionWebhookEnabled, - "admissionWebhookBindAddress", cfg.admissionWebhookBindAddress) + "auditInsecure", cfg.auditInsecure) setupLog.Info("Sensitive resource policy", "resources", cfg.sensitiveResources.Entries()) // Initialize metrics @@ -113,7 +108,7 @@ func main() { ) // Manager - mgr := newManager(metricsServerOptions, cfg.probeAddr, cfg, tlsOpts) + mgr := newManager(metricsServerOptions, cfg.probeAddr) // Expose build metadata on the metrics server so an operator can confirm a // running pod is the build they expect (also logged at startup above). @@ -184,6 +179,9 @@ func main() { // A worker discards a snapshot an apply made stale and asks for a fresh one, and asks no applier // for anything before a GitTarget's initial snapshots are in Git: both read the streams. workerManager.SetSnapshotHooks(watchMgr.InitialSnapshotsSettled, watchMgr.RequestRecheckForGitTarget) + // An apply window waits, after its operation completed, until the streams delivered every write + // made before then, however far behind attribution held them. + workerManager.SetApplyBarrier(watchMgr.ApplyBarrier) // The branch workers decide when a GitTarget's applier may apply; the dispatcher delivers it, // off their queues, to the applier its applier claim holds, in the applier claim's cluster. The status // writer records each operation on the applier claim, also off their queues. @@ -219,8 +217,8 @@ func main() { // Valkey/Redis is optional. When configured it holds each GitTarget's watch resume cursor so work // is re-picked up exactly where it left off after a restart or reconnect. When not configured the // WatchCursorStore stays nil and watches cold-replay from scratch on restart instead of resuming. - // Author attribution and the admission webhook both require Redis; validation has already rejected - // those combinations when redis-addr is empty. + // Author attribution over the Redis transport requires it; validation has already rejected that + // combination when redis-addr is empty. var redisStore *queue.RedisStore var redisGate *redisReadinessGate if cfg.redisAddr != "" { @@ -328,6 +326,18 @@ func main() { // Setup watch manager (must be after controllers are set up) fatalIfErr(watchMgr.SetupWithManager(mgr), "unable to setup watch ingestion manager") fatalIfErr(mgr.Add(watchMgr), "unable to add watch ingestion manager") + // CommitRequests created on a source cluster, next to the edits they save: served through every + // KubeProvider whose spec.commitRequests reconciles them, attributed from that cluster's audit facts. + commitRequestServer := &controller.CommitRequestServer{ + Client: mgr.GetClient(), + Clusters: watchMgr, + Finalizer: eventRouter, + Authors: watchMgr.AuthorResolver, + FactStreams: watchMgr.FactStreams, + TTL: cfg.commitRequestTTL, + Log: ctrl.Log.WithName("commitrequests"), + } + fatalIfErr(mgr.Add(commitRequestServer), "unable to add the CommitRequest server") // The GitBranch, GitTarget and applier claim controllers find a GitProvider's GitBranches, in any // namespace, through this index. @@ -357,6 +367,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), KubeConfigSafety: cfg.kubeConfigSafety, + CommitRequests: commitRequestServer, Recorder: mgr.GetEventRecorderFor("kubeprovider"), } // A nil registry must stay a nil INTERFACE, not one wrapping a nil pointer: the reconciler @@ -401,44 +412,21 @@ func main() { Follower: applyDispatcher, }).SetupWithManager(mgr), "unable to create controller", "controller", kind) } - // Command authorship is captured at admission by the validate-operator-types webhook and - // lives in its own Redis corner (author:v1:command), independent of - // --author-attribution (which governs mirrored-resource attribution). It is wired - // whenever the admission server is on; the controller reads the captured submitter - // back with no wait (docs/spec/commitrequest-admission-authorship.md). - // - // AuthorLookup must be a nil interface — not a non-nil interface wrapping a nil - // *CommandAuthorStore — when the webhook is off, so the controller's nil check - // selects the no-actor path immediately (AuthorAttributed=False) instead of - // dereferencing a nil store. - var commandAuthorStore *queue.CommandAuthorStore - var commandAuthorLookup controller.CommandAuthorLookup - if cfg.admissionWebhookEnabled { - if redisStore != nil { - commandAuthorStore = redisStore.CommandAuthorStore() - commandAuthorLookup = commandAuthorStore - setupLog.Info("validate-operator-types webhook enabled: command submitters are captured at admission " + - "and recorded as the request's named actor") - } else { - setupLog.Info("validate-operator-types webhook enabled without Redis: command author capture is a " + - "no-op; CommitRequests claim no actor (AuthorAttributed=False). " + - "Set --redis-addr to capture command authors.") - } - } + // A CommitRequest's author comes from its audit fact on the route of its GitTarget's + // KubeProvider, as a watched write's does, so the reconciler shares the watch plane's resolver + // and fact streams; both stay nil with attribution off. if err := (&controller.CommitRequestReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - APIReader: mgr.GetAPIReader(), - Finalizer: eventRouter, - AuthorLookup: commandAuthorLookup, - TTL: cfg.commitRequestTTL, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + APIReader: mgr.GetAPIReader(), + Finalizer: eventRouter, + Authors: watchMgr.AuthorResolver, + FactStreams: watchMgr.FactStreams, + TTL: cfg.commitRequestTTL, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CommitRequest") os.Exit(1) } - if cfg.admissionWebhookEnabled { - setupAdmissionWebhooks(mgr, commandAuthorStore) - } // +kubebuilder:scaffold:builder // Cert watchers (auditCertWatcher is nil in configured-author mode / --audit-insecure). @@ -465,11 +453,6 @@ type appConfig struct { metricsCertKey string probeAddr string metricsInsecure bool - admissionWebhookEnabled bool - admissionWebhookBindAddress string - admissionWebhookCertPath string - admissionWebhookCertName string - admissionWebhookCertKey string enableHTTP2 bool auditBindAddress string auditCertPath string @@ -536,20 +519,6 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { fs.StringVar(&cfg.probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") fs.BoolVar(&cfg.metricsInsecure, "metrics-insecure", false, "Serve the metrics endpoint over plain HTTP instead of HTTPS (default false; HTTPS).") - fs.BoolVar(&cfg.admissionWebhookEnabled, "admission-webhook", false, - "Serve the validating admission webhook endpoint (default false; off). When true, the webhook "+ - "server binds --admission-webhook-bind-address and requires --admission-webhook-cert-path.") - fs.StringVar(&cfg.admissionWebhookBindAddress, "admission-webhook-bind-address", defaultAdmissionWebhookBindAddr, - "Address (host:port) the validating admission webhook HTTPS server binds to; "+ - "an empty host (\":9443\") binds all interfaces.") - bindServerCertFlags( - fs, - "admission-webhook", - "validating admission webhook TLS", - &cfg.admissionWebhookCertPath, - &cfg.admissionWebhookCertName, - &cfg.admissionWebhookCertKey, - ) bindServerCertFlags( fs, "metrics", @@ -583,8 +552,7 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "and, with --author-attribution-transport=redis, the attribution fact streams. Leave empty to "+ "run without Redis: watches cold-replay on restart instead of resuming, and attribution needs "+ "--author-attribution-transport=memory. Required by --author-attribution-transport=redis when "+ - "attribution is on. --admission-webhook still runs without it, but command-author capture is a "+ - "no-op: CommitRequests claim no actor.") + "attribution is on.") fs.BoolVar(&cfg.authorAttribution, "author-attribution", true, "Name the real actor (human or service account) who caused each change as the Git commit author, "+ "resolved from matching audit facts; this runs the audit webhook ingress (default true). When "+ @@ -749,9 +717,6 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { if err := validateAuditConfig(cfg); err != nil { return appConfig{}, err } - if err := validateAdmissionWebhookConfig(cfg); err != nil { - return appConfig{}, err - } if err := validateReceiverConfig(cfg); err != nil { return appConfig{}, err } @@ -840,8 +805,7 @@ func validateAuditConfig(cfg appConfig) error { if strings.TrimSpace(cfg.redisAddr) == "" { // Attribution no longer implies Redis. It needs a fact transport, and the in-memory one is a // supported configuration rather than a rejected one — which is the whole reason the transport - // is selectable. Redis stays required for the transport that uses it, and for the admission - // webhook's command-author capture, which has no in-memory counterpart. + // is selectable. Redis stays required for the transport that uses it. if cfg.authorAttribution && cfg.attributionTransport == transportRedis { return errors.New( "redis-addr is required when author-attribution is enabled with " + @@ -922,22 +886,6 @@ func validateAttributionTransport(cfg appConfig) error { return nil } -func validateAdmissionWebhookConfig(cfg appConfig) error { - if !cfg.admissionWebhookEnabled { - return nil - } - // No redis-addr requirement here: the admission webhook stays enabled without Redis and - // simply no-ops command-author capture (CommitRequests claim no actor). Redis is - // wired only when configured; see the commandAuthorStore setup. - if _, _, err := splitBindAddress(cfg.admissionWebhookBindAddress); err != nil { - return fmt.Errorf("invalid admission-webhook-bind-address %q: %w", cfg.admissionWebhookBindAddress, err) - } - if strings.TrimSpace(cfg.admissionWebhookCertPath) == "" { - return errors.New("admission-webhook-cert-path is required when admission webhook is enabled") - } - return nil -} - // fatalIfErr logs and exits the process if err is not nil. func fatalIfErr(err error, msg string, keysAndValues ...any) { if err != nil { @@ -1232,35 +1180,11 @@ func loadCertPoolFromPEMFile(path string) (*x509.CertPool, error) { } // newManager creates a new controller-runtime Manager with common options. -func newManager( - metricsOptions metricsserver.Options, - probeAddr string, - cfg appConfig, - baseTLS []func(*tls.Config), -) ctrl.Manager { - var webhookServer ctrlwebhook.Server - if cfg.admissionWebhookEnabled { - // The bind address was validated in validateAdmissionWebhookConfig; controller-runtime's - // webhook server takes Host and Port separately, so split it back here. - host, port, err := splitBindAddress(cfg.admissionWebhookBindAddress) - if err != nil { - setupLog.Error(err, "invalid --admission-webhook-bind-address") - os.Exit(1) - } - webhookServer = ctrlwebhook.NewServer(ctrlwebhook.Options{ - Host: host, - Port: port, - CertDir: cfg.admissionWebhookCertPath, - CertName: cfg.admissionWebhookCertName, - KeyName: cfg.admissionWebhookCertKey, - TLSOpts: baseTLS, - }) - } +func newManager(metricsOptions metricsserver.Options, probeAddr string) ctrl.Manager { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsOptions, HealthProbeBindAddress: probeAddr, - WebhookServer: webhookServer, // Never cache Secret values. The control plane reads a small set of named // Secrets (Git credentials, signing keys, age keys) directly by name; caching // them would start a cluster-wide Secret informer that retains every Secret @@ -1281,27 +1205,6 @@ func newManager( return mgr } -// setupAdmissionWebhooks registers both handlers on the one admission server: the -// always-allow observer (a future-policy extension point) and the validate-operator-types -// handler, which captures the submitter of our own command kinds into commandAuthorStore and -// validates a WatchRule's source namespaces against the GitTarget it names. -func setupAdmissionWebhooks(mgr ctrl.Manager, commandAuthorStore *queue.CommandAuthorStore) { - mgr.GetWebhookServer().Register( - webhookhandler.ValidateAllPath, - &ctrladmission.Webhook{Handler: webhookhandler.AdmissionAllowHandler{}}, - ) - // Leave Store as a nil interface when there is no Redis-backed store, so the handler - // no-ops rather than dereferencing a typed-nil *CommandAuthorStore. - operatorTypesHandler := &webhookhandler.ValidateOperatorTypesHandler{Client: mgr.GetClient()} - if commandAuthorStore != nil { - operatorTypesHandler.Store = commandAuthorStore - } - mgr.GetWebhookServer().Register( - webhookhandler.ValidateOperatorTypesPath, - &ctrladmission.Webhook{Handler: operatorTypesHandler}, - ) -} - // addCertWatchersToManager attaches optional certificate watchers to the manager. func addCertWatchersToManager( mgr ctrl.Manager, diff --git a/cmd/main_audit_server_test.go b/cmd/main_audit_server_test.go index 25f689318..9e5e76369 100644 --- a/cmd/main_audit_server_test.go +++ b/cmd/main_audit_server_test.go @@ -33,8 +33,6 @@ func TestParseFlagsWithArgs_Defaults(t *testing.T) { require.NoError(t, err) assert.False(t, cfg.metricsInsecure) - assert.False(t, cfg.admissionWebhookEnabled) - assert.Equal(t, ":9443", cfg.admissionWebhookBindAddress) assert.False(t, cfg.auditInsecure) assert.Equal(t, "0.0.0.0:9444", cfg.auditBindAddress) assert.Equal(t, "/tmp/k8s-audit-server/audit-client-ca", cfg.auditClientCAPath) @@ -56,26 +54,6 @@ func TestParseFlagsWithArgs_Defaults(t *testing.T) { assert.False(t, cfg.credentialPolicy.AllowInsecureGitHTTP) } -func TestParseFlagsWithArgs_AdmissionWebhookValues(t *testing.T) { - fs := flag.NewFlagSet("test-admission-webhook", flag.ContinueOnError) - args := []string{ - "--admission-webhook", - "--admission-webhook-bind-address=:9445", - "--admission-webhook-cert-path=/tmp/admission-certs", - "--admission-webhook-cert-name=cert.pem", - "--admission-webhook-cert-key=key.pem", - } - - cfg, err := parseFlagsWithArgs(fs, args) - require.NoError(t, err) - - assert.True(t, cfg.admissionWebhookEnabled) - assert.Equal(t, ":9445", cfg.admissionWebhookBindAddress) - assert.Equal(t, "/tmp/admission-certs", cfg.admissionWebhookCertPath) - assert.Equal(t, "cert.pem", cfg.admissionWebhookCertName) - assert.Equal(t, "key.pem", cfg.admissionWebhookCertKey) -} - func TestParseFlagsWithArgs_AuditUnsecure(t *testing.T) { fs := flag.NewFlagSet("test-audit-insecure", flag.ContinueOnError) args := []string{ @@ -282,16 +260,10 @@ func TestParseFlagsWithArgs_InvalidAuditSettings(t *testing.T) { args: []string{"--additional-sensitive-resources=example.io/v1/credentials"}, }, { - name: "invalid admission webhook bind address port", - args: []string{"--admission-webhook", "--admission-webhook-bind-address=:0"}, - }, - { - name: "missing admission webhook cert path", - args: []string{"--admission-webhook", "--admission-webhook-cert-path="}, - }, - { - name: "admission webhook without redis", - args: []string{"--admission-webhook", "--admission-webhook-cert-path=/tmp/certs", "--redis-addr="}, + // The admission webhook is gone, and so are its flags: an install that still passes one + // fails at startup rather than running without what it asked for. + name: "the removed admission webhook flag", + args: []string{"--admission-webhook"}, }, } diff --git a/cmd/main_chart_certificates_test.go b/cmd/main_chart_certificates_test.go new file mode 100644 index 000000000..037e778d1 --- /dev/null +++ b/cmd/main_chart_certificates_test.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The default install mints no certificate, so it renders no cert-manager object at all, and installs +// on a cluster without cert-manager. The self-signed Issuer renders exactly when a certificate needs +// it: attribution with audit TLS, or metrics TLS. +func TestChart_TheIssuerRendersOnlyForACertificate(t *testing.T) { + _, err := os.ReadFile("../charts/gitops-reverser/templates/certificates.yaml") + require.NoError(t, err) + kinds := func(setValues ...string) []string { + var out []string + for _, template := range []string{"templates/certificates.yaml", "templates/audit-certificates.yaml"} { + for _, doc := range helmTemplateDocs(t, template, setValues...) { + for _, line := range strings.Split(doc, "\n") { + if kind, ok := strings.CutPrefix(line, "kind: "); ok { + out = append(out, kind) + } + } + } + } + return out + } + + assert.Empty(t, kinds(), "the default install needs no cert-manager") + assert.Contains(t, kinds("servers.metrics.tls.enabled=true"), "Issuer") + assert.Contains(t, kinds("attribution.enabled=true", "queue.redis.addr=valkey:6379"), "Issuer") + assert.NotContains(t, kinds("attribution.enabled=true", "queue.redis.addr=valkey:6379", + "servers.audit.tls.certManager=false"), "Issuer", "Secrets you bring yourself need no Issuer") +} diff --git a/cmd/main_redis_flags_test.go b/cmd/main_redis_flags_test.go index e04b8c766..0d65ac2bc 100644 --- a/cmd/main_redis_flags_test.go +++ b/cmd/main_redis_flags_test.go @@ -34,20 +34,6 @@ func TestParseFlags_RedisAddrIsOnlyRequiredByAttribution(t *testing.T) { "--audit-insecure", }, }, - // The webhook is failurePolicy: Ignore by design and the controller is the real gate, - // so running it without Redis is a supported, degraded mode — not a usage error. - // (--admission-webhook-cert-path has no default and is required whenever the webhook - // is on; that requirement is about TLS, not about Redis.) - "admission webhook runs without redis": { - args: []string{ - "--redis-addr=", "--author-attribution=false", - "--admission-webhook", "--admission-webhook-cert-path=/tmp/certs", - }, - }, - "admission webhook still requires its cert path": { - args: []string{"--redis-addr=", "--author-attribution=false", "--admission-webhook"}, - wantErr: "admission-webhook-cert-path is required", - }, "configured-author runs without redis": { args: []string{"--redis-addr=", "--author-attribution=false"}, }, diff --git a/config/certs/certificates.yaml b/config/certs/certificates.yaml index c9e4d2fd0..1f5c428c5 100644 --- a/config/certs/certificates.yaml +++ b/config/certs/certificates.yaml @@ -77,28 +77,3 @@ spec: usages: - digital signature - client auth ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: gitops-reverser - name: gitops-reverser-admission-server-cert - namespace: sut -spec: - dnsNames: - - gitops-reverser-service.sut.svc - - gitops-reverser-service.sut.svc.cluster.local - duration: 2160h - issuerRef: - kind: Issuer - name: gitops-reverser-audit-ca-issuer - privateKey: - rotationPolicy: Always - renewBefore: 360h - secretName: admission-server-cert - usages: - - digital signature - - key encipherment - - server auth diff --git a/config/crd/bases/configbutler.ai_argocdclaims.yaml b/config/crd/bases/configbutler.ai_argocdclaims.yaml index 0a5162e80..55d99bb1a 100644 --- a/config/crd/bases/configbutler.ai_argocdclaims.yaml +++ b/config/crd/bases/configbutler.ai_argocdclaims.yaml @@ -66,7 +66,9 @@ spec: - after a refused edit, on a GitTarget with spec.onRefusal "Reapply"; - after a sync reverted an edit made through the API; - after objects were created through the API in its folders; - - once at startup, and when a GitTarget first names this ArgoCDClaim. + - once at startup, and when a GitTarget first names this ArgoCDClaim; + - once spec.catchUpAfter passed since a push of GitOps Reverser's own; + - after a CommitRequest with spec.catchUp "AfterPush" pushed a commit. GitTargets name it with spec.applierClaimRef. properties: @@ -124,6 +126,26 @@ spec: rule: self == oldSelf - message: spec.applicationRef.name must not be empty rule: self.name != '' + catchUpAfter: + default: 0s + description: |- + CatchUpAfter is how long the Application stays behind the branch after + a push of GitOps Reverser's own before a sync of the tip is asked for, + as a Go duration ("5m"). Such a push changes nothing on the cluster, + which already holds the edit, but leaves the Application's last sync at + the commit before it. The interval starts when the Application is first + seen behind, so a burst of saves costs one sync, hooks included, and one + at the tip costs none. The sync can still wait for startup, a retry + backoff or edits being published. "0s" never asks for this reason, and + drops a sync it asked for that has not started. Otherwise at least "10s" + and at most "24h". Defaults to "0s". + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$ + type: string + x-kubernetes-validations: + - message: spec.catchUpAfter must be 0s, or a Go duration from 10s + to 24h + rule: duration(self) == duration('0s') || (duration(self) >= duration('10s') + && duration(self) <= duration('24h')) kubeProviderRef: description: |- KubeProviderRef names the KubeProvider of the cluster that runs the diff --git a/config/crd/bases/configbutler.ai_commitrequests.yaml b/config/crd/bases/configbutler.ai_commitrequests.yaml index 90b56c585..64e4b8ebd 100644 --- a/config/crd/bases/configbutler.ai_commitrequests.yaml +++ b/config/crd/bases/configbutler.ai_commitrequests.yaml @@ -24,17 +24,20 @@ spec: - jsonPath: .status.conditions[?(@.type=="Ready")].reason name: Reason type: string + - jsonPath: .status.author + name: Author + type: string - jsonPath: .status.commit name: Commit type: string - - jsonPath: .status.conditions[?(@.type=="AuthorAttributed")].status - name: AuthorAttributed - priority: 1 - type: string - jsonPath: .status.conditions[?(@.type=="Pushed")].status name: Pushed priority: 1 type: string + - jsonPath: .status.conditions[?(@.type=="Applied")].status + name: Applied + priority: 1 + type: string - jsonPath: .status.branch name: Branch priority: 1 @@ -52,8 +55,12 @@ spec: description: |- CommitRequest asks for a save: it attaches to its author's current or next commit window on a GitTarget, closes that window on its own timers, and - gives the commit its message. Its spec is immutable, and status.commit - reports the commit once it is on the remote. + gives the commit its message. Create it on the cluster GitOps Reverser runs + in, or on a source cluster whose KubeProvider sets spec.commitRequests. Its + spec is immutable, and status.commit reports the commit once it is on the + remote. + + Schema revision 1, compatible from revision 1. properties: apiVersion: description: |- @@ -77,14 +84,35 @@ spec: spec defines the desired state of CommitRequest. It is immutable after creation. properties: + catchUp: + default: AsDefinedOnClaim + description: |- + CatchUp decides when the GitTarget's applier is asked to apply the branch + after this request pushes a commit. "AsDefinedOnClaim" leaves it to the + applier claim's spec.catchUpAfter. "AfterPush" asks right after the push. + The apply takes the latest branch tip, which may hold later commits, and + saves pushed close together can share one apply; the Applied condition + reports how it went. The cluster already holds the saved objects: + the apply brings the applier's own state, such as a Kustomization's + status.lastAppliedRevision, to the tip. Defaults to "AsDefinedOnClaim". + enum: + - AsDefinedOnClaim + - AfterPush + type: string gitTargetRef: description: |- - GitTargetRef names the GitTarget, in this namespace, whose commit window - this request attaches to. + GitTargetRef names the GitTarget whose commit window this request + attaches to: in this namespace, or namespace names another. The + GitTarget's spec.allowedCommitRequests must admit this namespace, or the + request fails with reason "NotAllowed". properties: name: description: Name of the referent. type: string + namespace: + description: Namespace of the referent, when not specified it + acts as LocalObjectReference. + type: string required: - name type: object @@ -127,14 +155,14 @@ spec: window: default: attach: CurrentOrNext - attachTimeout: 2s + attachTimeout: 5s maxDuration: 2s description: |- Window says which commit window this request attaches to, how long it waits for one, and the timers that close the window once it has. They replace the GitTarget's timers for that window. A request never opens a window: only a write does. Omitted, the request attaches to the author's current or next - window within 2s and closes it 2s later. + window within 5s and closes it 2s later. properties: attach: default: CurrentOrNext @@ -149,13 +177,13 @@ spec: - Next type: string attachTimeout: - default: 2s + default: 5s description: |- AttachTimeout is how long the request waits for a window, as a Go duration, - from when GitOps Reverser picks the request up. "0s" attaches only to a - window already open. A request whose wait ran out never takes a later - window. At most "5m". Defaults to "2s", which covers a write's wait for its - audit event. + from when GitOps Reverser picks the request up, which is after its own + author was found. "0s" attaches only to a window already open. A request + whose wait ran out never takes a later window. At most "5m". Defaults to + "5s", which covers a write's wait for its audit event. pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$ type: string x-kubernetes-validations: @@ -203,7 +231,9 @@ spec: == has(oldSelf.message) && (!has(self.message) || self.message == oldSelf.message) && has(self.window) == has(oldSelf.window) && has(self.whenNothingToCommit) == has(oldSelf.whenNothingToCommit) && (!has(self.whenNothingToCommit) - || self.whenNothingToCommit == oldSelf.whenNothingToCommit) + || self.whenNothingToCommit == oldSelf.whenNothingToCommit) && has(self.catchUp) + == has(oldSelf.catchUp) && (!has(self.catchUp) || self.catchUp == + oldSelf.catchUp) - message: 'whenNothingToCommit: CommitEmpty requires spec.message, because a message is all an empty commit records' rule: '!has(self.whenNothingToCommit) || self.whenNothingToCommit != @@ -211,6 +241,20 @@ spec: status: description: status defines the observed state of CommitRequest. properties: + appliedRevision: + description: |- + AppliedRevision is the revision the GitTarget's applier reported applying + for this request, as in a Kustomization's status.lastAppliedRevision. It + holds the request's commit, and may be a later one. Set once Applied is + True with reason Applied. + type: string + author: + description: |- + Author is who created this request, as the audit events of the cluster + it was created on name them. Empty when they name nobody, or when author + attribution is off: then the request saves only writes that name nobody + either. + type: string branch: description: Branch is the branch the commit is made on. Set once the request ends. @@ -221,12 +265,31 @@ spec: type: string conditions: description: |- - Conditions are the latest observations of this CommitRequest's state. Ready, - Reconciling and Stalled follow kstatus: Ready is True once the request ended - without error, with a pushed commit or with nothing to save. Pushed is True - once the commit is on the remote. AuthorAttributed says whether the - request's creator, recorded at admission, names the commit author; False is - not a failure and does not affect Ready. + Conditions are the latest observations of this CommitRequest's state. + Reconciling and Stalled follow kstatus. + + Three conditions say how far the save got: + + - Ready (kstatus): Unknown while the save runs. True once the request + ended without error, with a pushed commit or with nothing to save; it + never waits for an apply. False when it failed: reason NotAllowed, + GitTargetNotFound, SaveInProgress or FinalizeFailed. + - Pushed: True once the commit is on the remote. + - Applied: set only with spec.catchUp "AfterPush", once Ready is True. + + Applied says whether the cluster's applier caught up with the commit: + + - Unknown, reason Applying or Retrying: the apply runs, or did not + complete and is asked again; the message says why. + - True, reason Applied: the applier applied a revision that holds the + commit, named in status.appliedRevision. + - True, reason NothingToApply, NoApplierClaim or ApplierGone: no apply + was needed. The request pushed no commit, or the GitTarget names no + applier claim, or stopped naming the one asked; the cluster already + holds the saved objects. + - False, reason CommitDropped: the branch no longer holds the commit; + its history was rewritten after the push. Only this is False. + - Unknown, reason Lost: GitOps Reverser restarted before it knew. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -285,6 +348,12 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + gitTargetUID: + description: |- + GitTargetUID is the UID of the GitTarget whose spec.allowedCommitRequests + admitted this request. A GitTarget with another UID, such as one created + after the request under the same name, must admit it again. + type: string observedGeneration: description: ObservedGeneration is the latest generation observed by the controller. diff --git a/config/crd/bases/configbutler.ai_fluxclaims.yaml b/config/crd/bases/configbutler.ai_fluxclaims.yaml index d05eb531b..385b5edb6 100644 --- a/config/crd/bases/configbutler.ai_fluxclaims.yaml +++ b/config/crd/bases/configbutler.ai_fluxclaims.yaml @@ -66,7 +66,9 @@ spec: - after a refused edit, on a GitTarget with spec.onRefusal "Reapply"; - after an apply reverted an edit made through the API; - after objects were created through the API in its folders; - - once at startup, and when a GitTarget first names this FluxClaim. + - once at startup, and when a GitTarget first names this FluxClaim; + - once spec.catchUpAfter passed since a push of GitOps Reverser's own; + - after a CommitRequest with spec.catchUp "AfterPush" pushed a commit. GitTargets name it with spec.applierClaimRef. properties: @@ -90,6 +92,27 @@ spec: spec: description: spec defines the desired state of FluxClaim. properties: + catchUpAfter: + default: 1m + description: |- + CatchUpAfter is how long the Kustomization stays behind the branch after + a push of GitOps Reverser's own before an apply of the tip is asked for, + as a Go duration ("1m", "90s"). Such a push changes nothing on the + cluster, which already holds the edit, but leaves + status.lastAppliedRevision and the GitRepository's artifact at the commit + before it. The interval starts when the Kustomization is first seen + behind, so a burst of saves costs one apply, and one at the tip costs + none. The apply can still wait for startup, a retry backoff or edits + being published. "0s" never asks for this reason, and drops an apply it + asked for that has not started. Otherwise at least "10s" and at most + "24h". Defaults to "1m". + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$ + type: string + x-kubernetes-validations: + - message: spec.catchUpAfter must be 0s, or a Go duration from 10s + to 24h + rule: duration(self) == duration('0s') || (duration(self) >= duration('10s') + && duration(self) <= duration('24h')) kubeProviderRef: description: |- KubeProviderRef names the KubeProvider of the cluster that runs the diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index f1c2572c4..84e81baa6 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -107,6 +107,180 @@ spec: spec: description: spec defines the desired state of GitTarget. properties: + allowedCommitRequests: + default: + - consumerNamespaces: + from: Same + description: |- + AllowedCommitRequests says which CommitRequests may save this GitTarget's + commit windows. Each grant sets one of: + + - consumerNamespaces: namespaces of the cluster GitOps Reverser runs + in. They admit requests only when spec.kubeProviderRef names an + inCluster KubeProvider. + - sourceNamespaces: namespaces of the cluster spec.kubeProviderRef + names, which must set spec.commitRequests. They admit requests only + when that KubeProvider names a kubeconfig. + + Omitted, the CommitRequests in this GitTarget's namespace are admitted, + as one grant with consumerNamespaces from: "Same". An empty list ([]) + admits none. A list replaces the default. A request no grant admits + fails with reason "NotAllowed". + + An admitted request saves its creator's whole commit window on this + GitTarget, writes outside the request's namespace included. + items: + description: |- + GitTargetCommitRequestGrant admits the CommitRequests of some namespaces to + save a GitTarget's commit windows: on the cluster GitOps Reverser runs in + (consumerNamespaces), or on the cluster the GitTarget mirrors + (sourceNamespaces). It sets exactly one of the two. + properties: + consumerNamespaces: + description: |- + ConsumerNamespaces are the namespaces, on the cluster GitOps Reverser + runs in, whose CommitRequests this grant admits. from: "Same" is this + GitTarget's namespace. It admits requests only for a GitTarget whose + KubeProvider sets spec.inCluster. + properties: + from: + description: |- + From selects which namespaces this grant allows consumers from: + + - "Same": the namespace of the object that holds the grant. + - "Names": the namespaces listed in names. + - "Selector": the namespaces whose labels match selector. + - "All": every namespace. + enum: + - Same + - Names + - Selector + - All + type: string + names: + description: 'Names are the namespaces allowed with from: + "Names", by exact name.' + items: + description: |- + GrantNamespace is a namespace a grant admits, by exact name. A variable and + "*" are refused. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: 'names: a variable is refused, because names + are matched against the consumer''s namespace and + ''{consumer.namespace}'' would admit every consumer; + use from: All to admit every namespace, or from: Same + for the namespace of the object holding the grant' + rule: self != '{consumer.namespace}' && self != '{consumer.name}' + - message: 'names: ''*'' is refused; use from: All to + admit every namespace' + rule: self != '*' + - message: names holds namespace names (DNS labels) + rule: self == '{consumer.namespace}' || self == '{consumer.name}' + || self == '*' || self.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-type: set + selector: + description: |- + Selector matches the labels of the namespaces allowed with from: + "Selector". It must not be empty: to allow every namespace, use from: + "All". + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - from + type: object + x-kubernetes-validations: + - message: 'from: Names needs names' + rule: self.from != 'Names' || has(self.names) + - message: 'names goes only with from: Names' + rule: '!has(self.names) || self.from == ''Names''' + - message: 'from: Selector needs a selector; to admit every + namespace, use from: All' + rule: self.from != 'Selector' || has(self.selector) + - message: 'selector goes only with from: Selector' + rule: '!has(self.selector) || self.from == ''Selector''' + - message: 'selector: {} is refused; to admit every namespace, + use from: All' + rule: '!has(self.selector) || (has(self.selector.matchLabels) + && size(self.selector.matchLabels) > 0) || (has(self.selector.matchExpressions) + && size(self.selector.matchExpressions) > 0)' + sourceNamespaces: + description: |- + SourceNamespaces are the namespaces, on the cluster spec.kubeProviderRef + names, whose CommitRequests this grant admits: exact names, or "*" for + every namespace this GitTarget may mirror. Variables are refused. A + namespace the KubeProvider does not lend this GitTarget in + spec.allowedGitTargets is never admitted. It admits requests only for a + GitTarget whose KubeProvider names a kubeconfig. + items: + description: |- + SourceNamespace is a namespace of a source cluster, by exact name, or "*" + for every namespace. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: a source namespace is a namespace name, or '*' + alone; variables are refused, because a source cluster's + namespaces are not the control cluster's + rule: self == '*' || self.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-type: set + type: object + x-kubernetes-validations: + - message: a grant in allowedCommitRequests sets exactly one of + consumerNamespaces and sourceNamespaces + rule: has(self.consumerNamespaces) != has(self.sourceNamespaces) + maxItems: 64 + type: array + x-kubernetes-list-type: atomic applierClaimRef: description: |- ApplierClaimRef names the FluxClaim or ArgoCDClaim, in this namespace, that diff --git a/config/crd/bases/configbutler.ai_kubeproviders.yaml b/config/crd/bases/configbutler.ai_kubeproviders.yaml index d32aa10b6..1487bc313 100644 --- a/config/crd/bases/configbutler.ai_kubeproviders.yaml +++ b/config/crd/bases/configbutler.ai_kubeproviders.yaml @@ -32,6 +32,10 @@ spec: name: Validated priority: 1 type: string + - jsonPath: .status.conditions[?(@.type=="CommitRequestsServed")].reason + name: Saves + priority: 1 + type: string - jsonPath: .status.conditions[?(@.type=="Ready")].message name: Message priority: 1 @@ -501,6 +505,34 @@ spec: minimum: 1 type: integer type: object + commitRequests: + default: Ignore + description: |- + CommitRequests says whether GitOps Reverser serves the CommitRequests + created on this cluster, for the GitTargets that mirror through this + KubeProvider: + + - "Ignore": they are left alone. + - "Reconcile": the requests in the namespaces this KubeProvider lends + its GitTargets in spec.allowedGitTargets are served, and their status + is written. A finished request stays. + - "ReconcileAndDeleteFinished": as "Reconcile", and a finished request + is annotated configbutler.ai/delete-after and deleted once that time + has passed. + + Defaults to "Ignore". Only "Ignore" goes with inCluster: the + CommitRequests of the cluster GitOps Reverser runs in are always served. + + The credential needs get, list and watch on commitrequests, and update + on commitrequests/status, in the configbutler.ai group, in each lent + namespace; "ReconcileAndDeleteFinished" also needs patch and delete on + commitrequests. The CommitRequest CRD must be installed on the cluster. + The CommitRequestsServed condition reports how it goes. + enum: + - Ignore + - Reconcile + - ReconcileAndDeleteFinished + type: string inCluster: description: |- InCluster makes this KubeProvider use the operator's own identity in the @@ -542,6 +574,11 @@ spec: recreate the KubeProvider to point it at a different cluster rule: has(self.inCluster) == has(oldSelf.inCluster) && (!has(self.secretRef) || self.secretRef == oldSelf.secretRef) + - message: 'spec.commitRequests must be Ignore beside spec.inCluster: + the CommitRequests of the cluster GitOps Reverser runs in are always + served' + rule: '!has(self.inCluster) || !has(self.commitRequests) || self.commitRequests + == ''Ignore''' status: description: status defines the observed state of KubeProvider. properties: @@ -551,6 +588,21 @@ spec: . when that is empty. The AuditFactsReceived condition describes this route, and starts over when it changes. type: string + commitRequests: + description: |- + CommitRequests is what this KubeProvider serves of its cluster's + CommitRequests. Absent while spec.commitRequests is "Ignore". + properties: + namespaces: + description: |- + Namespaces are the namespaces whose CommitRequests are watched: the ones + this KubeProvider lends the GitTargets that mirror through it. "*" is + every namespace, in one watch. + items: + type: string + type: array + x-kubernetes-list-type: set + type: object conditions: description: |- Conditions are the latest observations of this KubeProvider's state. Ready, @@ -566,6 +618,19 @@ spec: NoAuditDelivery (no audit events arrive at all), or RouteUnused (audit events arrive, but never on this route). It is absent when author attribution is off. + + CommitRequestsServed, which is not part of Ready either, says whether + this cluster's CommitRequests are served. It is absent while + spec.commitRequests is "Ignore". Each reason that is not True ends with + its fix: + + - True, Watching: watched in status.commitRequests.namespaces. + - True, NothingToWatch: no GitTarget mirroring through this KubeProvider + is lent a namespace. + - False, CRDNotInstalled: the cluster serves no CommitRequest CRD. + - False, VersionNotServed: the CRD does not serve v1alpha3. + - False, Forbidden: the credential may not list or watch them. + - Unknown, Unreachable: the cluster's API could not be read. items: description: Condition contains details for one aspect of the current state of this API Resource. diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index ce7cb62c6..330ed9864 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,11 +3,11 @@ kind: Kustomization resources: - bases/configbutler.ai_argocdclaims.yaml - bases/configbutler.ai_clusterwatchrules.yaml - - bases/configbutler.ai_commitrequests.yaml - bases/configbutler.ai_fluxclaims.yaml - bases/configbutler.ai_gitbranches.yaml - bases/configbutler.ai_gitproviders.yaml - bases/configbutler.ai_gitreceivers.yaml - bases/configbutler.ai_gittargets.yaml - bases/configbutler.ai_kubeproviders.yaml + - bases/configbutler.ai_commitrequests.yaml - bases/configbutler.ai_watchrules.yaml diff --git a/config/deployment.yaml b/config/deployment.yaml index 4cb63548c..77d1516d7 100644 --- a/config/deployment.yaml +++ b/config/deployment.yaml @@ -27,8 +27,6 @@ spec: - --metrics-bind-address=:8443 - --metrics-insecure - --health-probe-bind-address=:8081 - - --admission-webhook - - --admission-webhook-cert-path=/tmp/k8s-webhook-server/serving-certs - --audit-cert-path=/tmp/k8s-audit-server/audit-server-certs - --audit-client-ca-path=/tmp/k8s-audit-server/audit-client-ca - --redis-addr=valkey.valkey-e2e.svc.cluster.local:6379 @@ -95,9 +93,6 @@ spec: - containerPort: 9444 name: audit protocol: TCP - - containerPort: 9443 - name: webhook - protocol: TCP - containerPort: 8443 name: metrics protocol: TCP @@ -132,9 +127,6 @@ spec: - mountPath: /tmp/k8s-audit-server/audit-client-ca name: audit-client-ca readOnly: true - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: admission-webhook-certs - readOnly: true - mountPath: /var/gocoverage name: gocoverage securityContext: @@ -152,9 +144,6 @@ spec: - name: audit-client-ca secret: secretName: gitops-reverser-audit-root-ca - - name: admission-webhook-certs - secret: - secretName: admission-server-cert # Coverage output for a -cover image (GOCOVERDIR). emptyDir keeps the # pod compliant with the restricted PodSecurity profile; it survives a # container restart, so `task e2e-coverage-collect` flushes by signalling diff --git a/config/kustomization.yaml b/config/kustomization.yaml index 0eeba6aef..98ab3d539 100644 --- a/config/kustomization.yaml +++ b/config/kustomization.yaml @@ -9,7 +9,6 @@ resources: - receiver-service.yaml - deployment.yaml - certs -- webhook - kubeprovider-local.yaml images: - name: gitops-reverser @@ -40,26 +39,4 @@ replacements: select: kind: Certificate name: gitops-reverser-audit-server-cert - - fieldPaths: - - spec.dnsNames.0 - - spec.dnsNames.1 - options: - delimiter: . - index: 1 - select: - kind: Certificate - name: gitops-reverser-admission-server-cert - - fieldPaths: - - metadata.annotations.[cert-manager.io/inject-ca-from] - options: - delimiter: / - index: 0 - select: - kind: ValidatingWebhookConfiguration - name: gitops-reverser-validating-webhook - - fieldPaths: - - webhooks.0.clientConfig.service.namespace - select: - kind: ValidatingWebhookConfiguration - name: gitops-reverser-validating-webhook namespace: gitops-reverser diff --git a/config/samples/commitrequest.yaml b/config/samples/commitrequest.yaml index 7b9ce5468..bc12076ac 100644 --- a/config/samples/commitrequest.yaml +++ b/config/samples/commitrequest.yaml @@ -5,7 +5,13 @@ # reported in .status.commit once it has been pushed. # # Frontends typically use metadata.generateName so each "save" is a fresh -# object. The referenced GitTarget must live in the same namespace. +# object. The referenced GitTarget is in the same namespace unless +# gitTargetRef.namespace names another, whose spec.allowedCommitRequests must +# admit this one. The request's author comes from its own audit event. +# +# For a GitTarget that mirrors another cluster, create the request on that +# cluster instead, with gitTargetRef.namespace set, once its KubeProvider sets +# spec.commitRequests and the cluster has the CommitRequest CRD. apiVersion: configbutler.ai/v1alpha3 kind: CommitRequest metadata: @@ -24,8 +30,8 @@ spec: # opens. Next first closes the open window, under its own message. attach: CurrentOrNext # How long to wait for a window, from when the branch worker registers the - # request. "2s" covers a write's wait for its audit fact. - attachTimeout: "2s" + # request. "5s" covers a write's wait for its audit fact. + attachTimeout: "5s" # Close the attached window this long after the attach. Add idleTimeout (at # most maxDuration) to also close it after that much silence. maxDuration: "2s" diff --git a/config/service.yaml b/config/service.yaml index 9c5d0a6c0..bb7fa95c4 100644 --- a/config/service.yaml +++ b/config/service.yaml @@ -13,10 +13,6 @@ spec: port: 8443 protocol: TCP targetPort: 8443 - - name: webhook - port: 9443 - protocol: TCP - targetPort: 9443 selector: app.kubernetes.io/name: gitops-reverser control-plane: gitops-reverser diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml deleted file mode 100644 index 19451dfea..000000000 --- a/config/webhook/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- validating-webhook.yaml diff --git a/config/webhook/validating-webhook.yaml b/config/webhook/validating-webhook.yaml deleted file mode 100644 index 37830cbce..000000000 --- a/config/webhook/validating-webhook.yaml +++ /dev/null @@ -1,120 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - annotations: - cert-manager.io/inject-ca-from: sut/gitops-reverser-admission-server-cert - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: gitops-reverser - name: gitops-reverser-validating-webhook -# This ValidatingWebhookConfiguration is part of the kustomize SUT overlay (config/). -# It carries two webhooks served by the one admission server (port 9443, gated behind -# --admission-webhook): -# -# 1. validate-all.configbutler.ai — the broad '*' observer below. E2E-ONLY: -# it is intentionally NOT packaged in the Helm chart. The handler always allows -# (pure observation, see internal/webhook/admission_allow_handler.go); when a real, -# rejecting policy becomes product behavior, the chart must gain matching packaging -# and this note must change. -# 2. validate-operator-types.configbutler.ai — captures the submitter of our own command -# kinds (a CommitRequest today) at admission. This one IS product behavior and is -# packaged in the chart (charts/.../templates/validate-operator-types-webhook.yaml); the -# entry here is the e2e SUT's copy. See -# docs/spec/commitrequest-admission-authorship.md. -# -# The cert-manager.io/inject-ca-from annotation injects the admission server CA bundle -# for every webhook in this configuration. -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: gitops-reverser-service - namespace: sut - path: /validate-all - port: 9443 - # Ignore (not Fail) is mandatory while this matches '*': the webhook backend is - # the gitops-reverser pod itself, so a Fail policy on core resources would - # deadlock the cluster at bootstrap (the apiserver could not admit the pod that - # serves the webhook). When a real, rejecting policy lands, switch to Fail only - # together with a namespaceSelector that excludes kube-system and this pod's own - # namespace. - failurePolicy: Ignore - matchPolicy: Equivalent - name: validate-all.configbutler.ai - # Broad match for TESTING and as the future-policy extension point: observe - # writes to all real (top-level) resources, mirroring the intent of the audit - # capture policy (test/e2e/cluster/audit/policy.yaml). Two honest gaps vs that - # policy: - # * A ValidatingWebhookConfiguration matches by inclusion only — it cannot - # express "everything EXCEPT events/pods/leases/...", so this set is a - # superset of the audit set. The handler always allows, so the extra - # matches are harmless (just more requests observed). - # * resources: ['*'] is top-level only and deliberately excludes subresources, - # mirroring the audit policy dropping */status and apps/*/scale noise. - rules: - - apiGroups: - - '*' - apiVersions: - - '*' - operations: - - CREATE - - UPDATE - - DELETE - resources: - - '*' - scope: '*' - sideEffects: None - # 1s, not the apiserver max of 30s. failurePolicy is Ignore, so on a slow or - # unreachable backend the request is admitted anyway after the timeout — a - # large value only adds latency to every cluster write, it buys no safety. - timeoutSeconds: 1 -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: gitops-reverser-service - namespace: sut - path: /validate-operator-types - port: 9443 - # Ignore, not Fail: a missed author capture must degrade to the committer, never - # reject a user's CommitRequest. The handler also returns Allowed even when the Redis - # write errors, so the command's success never depends on this webhook at all — - # Ignore only covers the webhook being entirely unreachable. - # - # It stays Ignore now that the endpoint can also REJECT a WatchRule, because that - # rejection is feedback rather than enforcement: the writer's own precondition refuses - # the same configuration whether or not this webhook ran. Fail here would make every - # WatchRule write depend on the operator being up, to gate something already gated. - failurePolicy: Ignore - matchPolicy: Equivalent - name: validate-operator-types.configbutler.ai - # Narrow: our own command kinds (one rule per command kind), plus the one config kind - # this endpoint validates. The submitter of a command is captured into the command-author - # Redis corner and read back by the CommitRequest controller with no wait; a WatchRule is - # checked against the GitTarget it names and may be rejected. - rules: - - apiGroups: - - configbutler.ai - apiVersions: - - v1alpha3 - operations: - - CREATE - resources: - - commitrequests - scope: Namespaced - # UPDATE as well as CREATE: a rule can grow a second sourceNamespace long after it was - # created, and that edit is the same mistake. - - apiGroups: - - configbutler.ai - apiVersions: - - v1alpha3 - operations: - - CREATE - - UPDATE - resources: - - watchrules - scope: Namespaced - # Redis write on real requests; nothing on dry-run (the handler honors NoneOnDryRun). - sideEffects: NoneOnDryRun - timeoutSeconds: 2 diff --git a/docs/INDEX.md b/docs/INDEX.md index a91139a69..f6287a1a3 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -103,8 +103,7 @@ misled. Full list in [`spec/README.md`](spec/README.md); the ones that carry a | [`gitpath-foreign-content-stringency.md`](spec/gitpath-foreign-content-stringency.md) | refusing a path that shadows foreign content | | [`unsupported-folder-refusal-plan.md`](spec/unsupported-folder-refusal-plan.md) | `GitPathAccepted`, and refusing what we cannot own | | [`commitrequest-design.md`](spec/commitrequest-design.md) | the CommitRequest window and its conditions | -| [`commitrequest-admission-authorship.md`](spec/commitrequest-admission-authorship.md) | how command submitters are captured and matched to commit windows | -| [`where-validation-lives.md`](spec/where-validation-lives.md) | schema → CEL → **the reconciler**; a webhook only for what exists solely at admission | +| [`where-validation-lives.md`](spec/where-validation-lives.md) | schema → CEL → **the reconciler**; no admission webhook ships | | [`e2e-serial-registry.md`](spec/e2e-serial-registry.md) | which e2e specs must run Serial, and why | ## What is being decided now — [`design/`](design/) @@ -153,6 +152,7 @@ so you never have to read a proposal to find out it already shipped. | [`watches-labels-simplification.md`](design/watches-labels-simplification.md) | **step 2, implemented.** After the collection rename and operation-filter removal, native per-rule `objectSelector` (#146). Kubernetes is authoritative: `Always` compares selected identities with all managed Git identities in the type/namespace scope, without comparing Git labels. Full support includes supported Kustomize layouts from the first release, plus label-exit attribution (now selector-agnostic, see the attribution spec) and ordered producer replacement. | | [`label-selection-follow-ups.md`](design/label-selection-follow-ups.md) | **design, open.** What the label-selection review rounds left open, with the evidence for each: real deletions in a selected collection wait the full 3s attribution grace (the watch cannot tell them from a label exit, measured), the resync fence that stops overlapping collections from coalescing (kept on purpose), same-second overlap ties, and one store rebuild per object coming back. Also lists what was fixed on the branch | | [`attribution-removal-wait-options.md`](design/attribution-removal-wait-options.md) | a removal now waits for evidence about the DELETION rather than accepting the object's last write, which stopped it naming whoever last edited the object as the author of a deletion they did not perform. Enumerates the eight situations a resolution can be in and shows the cost is concentrated in exactly one: a removal for which no delete fact will ever arrive (a graceful pod delete, a status-only removal, a type the audit policy skips) spends the whole grace to return the answer it had at t=0, measured at ~3.1s against ~70ms when evidence is present. Prices five options against that, and recommends a per-route watermark — stop waiting once the fact stream has demonstrably moved past this event — over a second timeout flag whose right value lives in the API server's config rather than ours. Open: the decision, and how common the case is outside the e2e suite | +| [`concurrent-author-windows.md`](design/concurrent-author-windows.md) | **proposal, not built.** A branch worker holds one commit window at a time, so a second person's edit closes the first person's window and splits their change into two commits. Proposes a window per GitTarget and author, open side by side, with one rule that keeps every path's history in order: a write to a path another open window holds closes that window first (`path_conflict`). Branch-wide closes (foreign commit, apply, shutdown, buffer limit) close every window. Commit order becomes close order. A second save of one author stays refused (`SaveInProgress`). Open: what `buffer_limit` closes, windows of several GitTargets side by side, a cap on open windows, and a save cut short by a path conflict | | [`commitrequest-save-wait-options.md`](design/commitrequest-save-wait-options.md) | **design, decided** — a five-phase plan, reported from a live cluster: a `CommitRequest` can expire before the write it exists to publish arrives, because the close delay defaulted to `0` while the write is still held head-of-line waiting for an audit fact, so the save resolves `Ready=True`/`NoWindowInGrace` and the edit commits seconds later under the target's `liveTemplate`. The decision is to repair the default (`spec.closeDelay`, now a duration string defaulting to `"2s"`, **shipped as phase 0**) and then let a request **name the write it is waiting for**, reusing the `(uid, resourceVersion)` identity the attribution join already keys on, including its settled `ExactCapable` rule. Records why the fact index cannot BE the wait (it sits upstream of the watch event, so a fact-keyed wait fires before the window opens) and what to take from it anyway — the register-then-check waiter discipline, and a best-effort retrospective oracle bounded by a TUNABLE TTL. Twelve situations, and the page is deliberate about the two it cannot fix cheaply: row 11 (a saturated queue is rejected in the WATCH path and never reaches the worker, so no worker-side wait can diagnose it) is an accepted, named gap rather than a promise, and row 10 is fixed only by a hook in the fidelity-drop branch that marks a wait terminally UNSATISFIABLE — satisfying before that gate would arm a finalize for a write that never commits. Carries a **preconditions chapter** for saves on today's build (audit policy first, then delay sizing, fidelity warm-up, save-button debounce, and assert on the commit rather than on `Ready`). `CommitterActivity` is deferred, not recommended: by its own arrival-not-existence rule it cannot satisfy the already-arrived case. Open: situation 12 (one actor, two saves, one window — blocks phase 2), whether the fact-index lookup earns its coupling, and how an unsatisfied wait is graded | | [`commit-timing-surface.md`](design/commit-timing-surface.md) | **built.** One configuration surface for commit windows and saves, an in-place `v1alpha3` break with one tested upgrade procedure: only writes open a window; a `window` block (`idleTimeout`, `maxDuration`, target default `1m`) on both kinds, which a `CommitRequest` replaces whole for the window it attaches to; `attach: CurrentOrNext \| Next` with `attachTimeout`; opt-in `whenNothingToCommit: CommitEmpty` that keeps the cause in the reason. Precise clock and zero rules. Supersedes the save-wait timer model. Open: the target's `maxDuration` value | | [`watch-and-catalog-architecture.md`](design/watch-and-catalog-architecture.md) | the target three-layer watch model — **needs a human call before building** | diff --git a/docs/TODO.md b/docs/TODO.md index 4519174bd..80b41eff9 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -116,8 +116,9 @@ Their argument for each is in [gitops-api-open-asks.md](future/gitops-api-open-a `Invalidate` re-issues one (`internal/git/render_fidelity_gate.go`), and close the retention revision-mismatch question in [watch-plane-status-convergence-failures.md](design/watch-plane-status-convergence-failures.md) §3. -- [ ] Filter more cluster-generated noise, such as `kube-root-ca.crt`; decide which objects and - whether a user can opt back in (`resource_policy.go` is a type-level denylist only). +- [ ] Filter more cluster-generated noise beyond `kube-root-ca.crt` (always left out, no opt-in: + `internal/watch/cluster_generated.go`), such as OpenShift's `openshift-service-ca.crt`, once + measured on such a cluster. - [ ] A manifest whose CRD is missing refuses the whole folder (`IssueUnresolvedKRM`, `manifestanalyzer/acceptance.go`): keep that, or defer only that document. The scan already records `unresolved-mapping` instead of failing. diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 176c54eaa..6c1ae59df 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -7,6 +7,62 @@ guidance that the changelog's breaking-change entries link to. We are pre-1.0, so breaking changes bump the **minor** version (release-please is configured with `bump-minor-pre-major`) rather than the major. Read the relevant entry before upgrading across it. +## The admission webhook is gone; a `CommitRequest`'s author comes from audit + +GitOps Reverser installs no admission webhook. The chart no longer renders the +`ValidatingWebhookConfiguration`, its `Certificate` and Secret, or port `9443` on the controller +`Service`, and the command-author records in Redis (`author:v1:command:*`) are no longer written; +those that remain expire on their own. + +- **Values and flags.** Drop `servers.admission` from your values, and the `--admission-webhook`, + `--admission-webhook-bind-address` and `--admission-webhook-cert-*` flags from a hand-written + Deployment: the controller refuses to start with one of them. +- **cert-manager is needed only for a certificate.** The chart renders its self-signed `Issuer` only + when it mints one: the audit ingress certificates with `attribution.enabled`, or the metrics + certificate with `servers.metrics.tls.enabled`. The default install needs no cert-manager. +- **A request's author.** A `CommitRequest`'s author comes from its own audit event, as a write's + does, and is reported in `status.author`. The `AuthorAttributed` condition is gone. Keep `create` + of `commitrequests` in your audit policy; the write catch-all of the + [tuned policy](../test/e2e/cluster/audit/policy.yaml) covers it. A request no audit event names, on + a cluster that sends audit events, ends `Ready=True` with reason `AuthorUnresolved` and saves + nothing. With attribution off, a request names nobody, as it did with the webhook off. +- **Status while it runs.** A `CommitRequest` in progress is `Ready=Unknown`, reason `Saving` until the + branch worker reports a phase, rather than `Ready=False`. Automation that waits for `Ready=True` or + `Stalled=True` is unaffected. +- **`spec.window.attachTimeout` defaults to `5s`**, since the request now waits for its own audit event + as its edits do. A request that sets it keeps its value. +- **A request is registered once its author is found**, no longer at creation. A client that creates a + request and then makes the edit it saves, on a `GitTarget` whose windows close on the first silence + (`idleTimeout: "0s"`), waits for `Ready=Unknown` before the edit, or the edit can commit alone. +- **A `GitTarget` that mirrors another cluster.** A `CommitRequest` on the cluster GitOps Reverser runs + in, for such a `GitTarget`, is refused with reason `NotAllowed`. Create it on the cluster the + `GitTarget` mirrors instead, as described in + [serving the `CommitRequest`s of a source cluster](configuration.md#serving-the-commitrequests-of-a-source-cluster). + +A `CommitRequest` may also name a `GitTarget` in another namespace now, in +`spec.gitTargetRef.namespace`, which that `GitTarget` admits in `spec.allowedCommitRequests`. Omitted, +the list admits the `GitTarget`'s own namespace, as before. + +## A `FluxClaim` applies the tip a minute after GitOps Reverser's own pushes + +A push of GitOps Reverser's own asks no apply at once, and in 0.54.0 asked none at all, so a +`Kustomization`'s `status.lastAppliedRevision` and its `GitRepository`'s artifact stayed at the last +commit somebody else pushed. `spec.catchUpAfter` sets when that lag is caught up: once the applier +has stayed behind the tip that long, an apply of the tip is asked for, one for a whole burst of +saves. A `FluxClaim` defaults to `1m`; set `0s` to keep 0.54.0's behaviour. An `ArgoCDClaim` +defaults to `0s`. A `CommitRequest` that sets `catchUp: AfterPush` has the tip applied right after +its commit is pushed, whatever the claim's interval, and reports the outcome in its `Applied` +condition and `status.appliedRevision`. + +## `kube-root-ca.crt` is not mirrored + +The `kube-root-ca.crt` ConfigMap the cluster writes into every namespace is never mirrored, whatever +a `WatchRule` selects: it holds that one cluster's CA, and the cluster overwrites any other value. +A copy already in Git is left like any document the cluster does not hold. Under prune mode +`Always` the next resync removes it; under `OnEvent` (the default) delete it from Git yourself. If an +applier applied that folder, it prunes the ConfigMap when the file goes, and the cluster recreates +it at once. + ## Upgrading from 0.53.0: recreate the connection objects in one pass **Breaking, with no in-place conversion.** The objects that connect a folder to a cluster, a diff --git a/docs/architecture.md b/docs/architecture.md index 50c1e6879..4b4bc0f99 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -239,13 +239,12 @@ optional and only add *who* did something: |---|---|---|---| | **Discovery** (CRD/APIService) | *what types exist* | yes | the API surface: served GVRs, scope, preferred version, subresources; rules resolve against it | | **Watch** (per claimed `(GVR, scope)`) | *what changed* | yes | the object body, ordered per type; the only object-state source, with deletes reconciled via `sendInitialEvents` replay + mark-and-sweep | -| **Audit webhook** (`/audit-webhook/`) | *who changed mirrored state* | no | a post-persist attribution fact, partitioned by source provider and joined to the watch event by resourceVersion ([Optional Attribution](#optional-attribution)) | -| **Validating admission webhook** (`/validate-operator-types`) | *who issued a command* | no | the submitter of a `CommitRequest`, captured at admission and keyed 1:1 by UID ([CommitRequest Finalize](#commitrequest-finalize)) | +| **Audit webhook** (`/audit-webhook/`) | *who changed mirrored state, and who created a `CommitRequest`* | no | a post-persist attribution fact, partitioned by source provider and joined to the watch event, or to the `CommitRequest`, by resourceVersion ([Optional Attribution](#optional-attribution)) | -With both optional sources off, the product still mirrors state correctly, and every commit is authored +With the optional source off, the product still mirrors state correctly, and every commit is authored by the committer. -### Why audit, not admission, attributes a mirrored change +### Why audit, not admission, attributes a change For a mirrored change, taking the author from a validating admission webhook is tempting, because the request already carries `userInfo`. We tried it; the edge cases are not fixable, because **admission runs *before* @@ -263,13 +262,11 @@ the write reaches etcd**: Audit avoids both: kube-apiserver posts a `ResponseComplete` event *after* the write persisted, carrying the resourceVersion that joins it to the watch event. -The one place admission *is* the right source is the `CommitRequest`, and only for its **command** meaning -(*who issued the save*) and not for mirroring the object. That holds because it is **our own type with a -memory model we control**: the controller reconciles off its own watch, which only delivers persisted -objects, so the author is captured at admission and read back 1:1 by UID: no resourceVersion join, no -persistence proof from audit needed. (We still can't guarantee a create is never blocked, but it holds -across our edge cases.) If a `WatchRule` also selects CommitRequests, that *mirroring* runs through the -normal pipeline and is audit-attributed like any other resource. See +A `CommitRequest`'s author comes from audit too, for the same reasons: its own `ResponseComplete` event, +joined by the UID and resourceVersion it was created with, from the same index and with the same grace +as the edits it saves. So the request and its edits are attributed by one pipeline, on the cluster both +were made on, and a request on a source cluster needs nothing from the control cluster. A +`CommitRequest` is never mirrored, whatever a `WatchRule` selects. See [CommitRequest Finalize](#commitrequest-finalize). ### Proving it on new Kubernetes versions @@ -407,24 +404,27 @@ One `KubeProvider` grant answers two questions in different planes: ### CommitRequest - **Source**: [api/v1alpha3/commitrequest_types.go](../api/v1alpha3/commitrequest_types.go) -- **Controller**: [internal/controller/commitrequest_controller.go](../internal/controller/commitrequest_controller.go) +- **Core**: [internal/controller/commitrequest_core.go](../internal/controller/commitrequest_core.go) -A one shot "save now" signal that finalizes the open commit window for a same namespace `GitTarget` -instead of waiting for the silence timer. The **entire spec is immutable**. Key fields: +A one shot "save now" signal that finalizes the open commit window for a `GitTarget` instead of waiting +for the silence timer. It is created on the cluster the `GitTarget` mirrors: the control cluster, or a +source cluster whose `KubeProvider` sets `spec.commitRequests`. The **entire spec is immutable**. Key +fields: -- `spec.gitTargetRef.name`: target whose window the request attaches to. +- `spec.gitTargetRef`: target whose window the request attaches to, admitted by its + `spec.allowedCommitRequests`. On a source cluster the namespace is required. - `spec.message`: optional verbatim commit message (1–1024 chars, no control characters). -- `spec.window`: `attach` (`CurrentOrNext` or `Next`), `attachTimeout` (default `2s`, the wait for a +- `spec.window`: `attach` (`CurrentOrNext` or `Next`), `attachTimeout` (default `5s`, the wait for a window), and the timers that replace the target's for the attached window: `idleTimeout` (none by default) and `maxDuration` (default `2s`). A request never opens a window; only writes do. - `spec.whenNothingToCommit`: `Resolve` (default) or `CommitEmpty`, which records the message in an empty commit when the request ends with nothing to commit. -- `status.conditions`: kstatus-compatible. **Ready** is the summary (True once the request reached a - terminal outcome that is not an error: a pushed commit, or a benign no-commit); - **Reconciling**/**Stalled** are the kstatus progress/blocked pair; **AuthorAttributed** reports - whether the `/validate-operator-types` validating admission webhook named the submitter; an absent record - means the request claims no actor and can attach only to an unnamed window. **Pushed** reports whether the - commit reached the remote. The `Ready` +- `status.author`: the creator, as the request's audit event names them; empty when it names nobody, + and then the request can attach only to an unnamed window. +- `status.conditions`: kstatus-compatible. **Ready** is the summary (Unknown while the save runs, True + once the request reached a terminal outcome that is not an error: a pushed commit, or a benign + no-commit); **Reconciling**/**Stalled** are the kstatus progress/blocked pair. **Pushed** reports + whether the commit reached the remote. The `Ready` condition's `reason` carries `Committed`, `NoWindow`, `WindowMismatch`, `AlreadyPresent`, or `FinalizeFailed`. A benign no-commit (e.g. `NoWindow`) is `Ready=True`, `Stalled=False`, a correct non-error outcome, whereas a `FinalizeFailed` is `Ready=False`, `Stalled=True`. @@ -743,8 +743,8 @@ flowchart TD TIMER -->|silence timer| BASE TIMER -->|buffer limit or resync boundary| BASE - CR[CommitRequest persisted] --> ADMISSION[Admission submitter lookup
named or unnamed] - ADMISSION --> ATTACH[Attach to matching open window] + CR[CommitRequest persisted] --> AUTHOR[Author from its audit fact
named or unnamed] + AUTHOR --> ATTACH[Attach to matching open window] ATTACH -->|same author + GitTarget| BASE ATTACH -->|no matching window| BENIGN[Ready=True, Pushed=False] @@ -1114,9 +1114,8 @@ queued behind it on the same `(GitTarget, GVR, scope)` goroutine. That cost, and is in [Watch Event Ordering](#watch-event-ordering) and [attribution-removal-wait-options.md](design/attribution-removal-wait-options.md). -The CommitRequest controller does **not** read this audit index. A CommitRequest's submitter is named by -the `/validate-operator-types` validating admission webhook instead, captured synchronously at admission, -never joined from an audit fact (see [CommitRequest Finalize](#commitrequest-finalize)). +A `CommitRequest`'s author is read from this index too, on the route of its `GitTarget`'s `KubeProvider`, +with the same grace (see [CommitRequest Finalize](#commitrequest-finalize)). ### Author and committer identity in Git @@ -1187,12 +1186,10 @@ CommitDate: Mon Jun 30 12:00:05 2026 +0000 ### Wiring OIDC author claims - **Audit-path extractor**: [internal/queue/audit_event_parsing.go](../internal/queue/audit_event_parsing.go) -- **Admission-path extractor**: - [internal/webhook/validate_operator_types_handler.go](../internal/webhook/validate_operator_types_handler.go) - **Signature construction**: [internal/git/commit.go](../internal/git/commit.go) (`authorName` / `authorEmail`) The operator never talks to the identity provider. It reads two fixed keys from the request's `user.extra` -map: the audit attribution path and the CommitRequest admission path resolve the **same** two keys: +map, from the audit event that names the actor: | `user.extra` key | OIDC claim it carries | Used for | |---|---|---| @@ -1760,44 +1757,53 @@ unsafely: logged per-resource, counted in the resync summary as `placementSkippe ## CommitRequest finalize -- **Controller**: [internal/controller/commitrequest_controller.go](../internal/controller/commitrequest_controller.go) -- **Admission handler**: - [internal/webhook/validate_operator_types_handler.go](../internal/webhook/validate_operator_types_handler.go) -- **Admission author cache**: - [internal/queue/command_author_store.go](../internal/queue/command_author_store.go) - -A `CommitRequest` finalizes the open commit window for its GitTarget. The request author is resolved from -the `/validate-operator-types` validating admission webhook, which captures the authenticated submitter at -admission (keyed by the object's UID) before the object is visible. The lookup is **present-or-never**: if -the record is missing, the request is marked `AuthorAttributed=False`, claims no actor, and still attaches -immediately. That is not a failure: - -1. The controller stamps the in-progress conditions (`Reconciling=True`) and settles - `AuthorAttributed` synchronously from the admission author cache. There is no audit wait on this path. -2. The controller eagerly **attaches** the request to the worker (`AttachCommitRequest`), which - waits for a window until `registration + attachTimeout`. The worker binds it to an open window only - when the author state and GitTarget match, first come first served, and replaces the window's timers - with the request's own, so time spent waiting never shortens the collection. It **never finalizes - another author's window**; a window carries at most one request. The controller reports the phase the - worker reports: `WaitingForWindow`, `CollectingWindow`, `WaitingForPush`. -3. The window finalizes when the request's timers close it (or when it closes for any other reason). +- **Core**: [internal/controller/commitrequest_core.go](../internal/controller/commitrequest_core.go) +- **Control-cluster path**: [internal/controller/commitrequest_controller.go](../internal/controller/commitrequest_controller.go) +- **Source-cluster path**: [internal/controller/commitrequest_server.go](../internal/controller/commitrequest_server.go) + +A `CommitRequest` finalizes the open commit window for its GitTarget. It is picked up in one of two +ways: by the `CommitRequestReconciler` on the control cluster, or by the `CommitRequestServer`'s watch +of a source cluster, through the `KubeProvider` whose `spec.commitRequests` reconciles it. Both hand it +to one core, which takes one step at a time, each decided again from the object as it is stored +([contract](spec/commitrequest-design.md)): + +1. **The owner.** The `GitTarget`'s `KubeProvider` alone picks the path that serves the request: the + control-cluster path for an `inCluster` provider, that provider's watch for a kubeconfig. Any other + path writes nothing but `GitTargetNotFound`, once `attachTimeout` has passed with no `GitTarget`, + and only into an empty status. Every status write is conditional on the resourceVersion it was + decided from. +2. **Admission and the author.** A grant in the `GitTarget`'s `spec.allowedCommitRequests` must admit + the request: `consumerNamespaces` on the control cluster, `sourceNamespaces` on a source cluster. + The request's author comes from its own audit fact, on the route of the `GitTarget`'s + `KubeProvider`, awaited with the attribution grace; a request no fact names, on a provider that has + received audit facts, is `AuthorUnresolved` and saves nothing. +3. **The claim.** The owner writes the first status, `status.author` and `Ready=Unknown` (`Saving`), + and only then **attaches** the request to the worker (`AttachCommitRequest`), which waits for a + window until `registration + attachTimeout`. The worker binds it to an open window only when the + author state and GitTarget match, first come first served, and replaces the window's timers with the + request's own, so time spent waiting never shortens the collection. It **never finalizes another + author's window**; a window carries at most one request. The owner reports the phase the worker + reports: `WaitingForWindow`, `CollectingWindow`, `WaitingForPush`. +4. The window finalizes when the request's timers close it (or when it closes for any other reason). If a finalize closes an open window, the worker always schedules a push, so a window closed by an otherwise no-op resync is not stranded. -4. Outcomes resolve on push and are reported as conditions: a pushed commit sets `Ready=True` / +5. Outcomes resolve on push and are reported as conditions: a pushed commit sets `Ready=True` / `Pushed=True` with `branch`/`commit`; a benign no-commit sets `Ready=True` with the reason on `Ready` and `Pushed=False`; a failure sets `Ready=False` / `Stalled=True` with a message. **Between the window's - finalize and that push the request is still in flight**, which matters because the controller keeps + finalize and that push the request is still in flight**, which matters because the owner keeps re-sending its attach until it reads an outcome: the worker marks the request committed rather than forgetting it, so a re-send is recognized as the same request and cannot resolve early or claim a later window. A worker that stops without pushing fails the request instead of leaving it to time out. `AlreadyPresent` therefore means the remote confirmed there was nothing to add, not that a local plan - found no diff. + found no diff. A conflict on the outcome's write re-applies the outcome onto the fresh object; the + save is never restarted. +6. **Retention.** A finished request is stamped `configbutler.ai/delete-after` and deleted once it has + passed: always on the control cluster, and on a source cluster only with + `commitRequests: ReconcileAndDeleteFinished`. -The CommitRequest submitter is not recoverable from object state alone, so without an admission record the -request cannot claim an actor. The final Git author remains the attached watch window's author: configured -committer when attribution was disabled, or the explicit unresolved author when live attribution ran but -could not resolve. There is no audit-fact join for the request itself: its submitter is captured at admission -by the validating webhook, not derived from the audit attribution index. +A request whose audit event names nobody claims no actor. The final Git author remains the attached +watch window's author: configured committer when attribution was disabled, or the explicit unresolved +author when live attribution ran but could not resolve. *** @@ -1830,10 +1836,12 @@ Controllers watch their dependencies so dependents reconcile quickly after spec at all (the installation's `inCluster` consent, and which namespace holds its audit route), and projects its readiness to dependent targets. A provider created, deleted or moved to another route wakes the others carrying that route, so an `AuditRouteConflict` clears without waiting for a requeue. -- `CommitRequestReconciler` runs with `MaxConcurrentReconciles=1` and attributes/attaches as above; its - optional `AuthorLookup` is the command-author cache populated by the `/validate-operator-types` webhook - (wired whenever the admission webhook is enabled, independent of `--author-attribution`, and nil - otherwise, so the request claims no actor). +- `CommitRequestReconciler` serves the control cluster's `CommitRequest`s through the shared core, four + at a time, so one request's author wait does not hold up the others. It reads through the uncached + API reader, and shares the watch plane's author resolver and fact streams, which are nil with + attribution off. The `CommitRequestServer` serves the source clusters' requests through the same core, + one watch per `KubeProvider` that reconciles them, and reports each to the `KubeProviderReconciler`, + which writes it as `CommitRequestsServed`. Dependency watches use narrow predicates to avoid status-only heartbeat churn: a `KubeProvider` also admits a `Ready` transition, and a `Namespace` admits only label changes. `GitProvider`, `GitBranch`, @@ -1868,7 +1876,7 @@ flowchart TD Hq -->|no| J[Configured-author: no fact streams or index; audit webhook skipped] I --> K[Setup + register Watch Manager] J --> K - K --> L[Register GitProvider + GitReceiver + KubeProvider + GitBranch + GitTarget
+ FluxClaim + ArgoCDClaim + CommitRequest controllers; admission webhooks] + K --> L[Register GitProvider + GitReceiver + KubeProvider + GitBranch + GitTarget
+ FluxClaim + ArgoCDClaim + CommitRequest controllers; CommitRequest server] L --> M[Add cert watchers + health checks] M --> N[mgr.Start] ``` @@ -2016,7 +2024,7 @@ Current limitations: | [internal/types/](../internal/types/) | shared resource identity/reference and sensitivity policy | | [internal/typeset/](../internal/typeset/) | type followability registry, lookup model, and the relevance filter (controller-owned → don't mirror) | | [internal/watch/](../internal/watch/) | discovery catalog, watch manager, per-`(GVR, scope)` raw watches with `sendInitialEvents` replay + mark-and-sweep, the author resolver, watched type tables, event router | -| [internal/webhook/](../internal/webhook/) | `/audit-webhook` ingress and attribution fact extraction (no body joiner), the `/validate-operator-types` admission handler, and the `/git-push` receiver | +| [internal/webhook/](../internal/webhook/) | `/audit-webhook` ingress and attribution fact extraction (no body joiner), and the `/git-push` receiver | *** diff --git a/docs/attribution-setup-guide.md b/docs/attribution-setup-guide.md index 5e3ff5120..62341e03a 100644 --- a/docs/attribution-setup-guide.md +++ b/docs/attribution-setup-guide.md @@ -26,8 +26,8 @@ queue: Redis/Valkey holds attribution facts and watch cursors. If its data is lost, recent facts are lost and watches cold-replay. For a disposable single-pod setup, `attribution.transport: memory` needs -no Redis but loses facts whenever the reverser restarts. It also leaves Redis-backed features such -as `CommitRequest` author capture inactive unless you configure Redis separately. +no Redis but loses facts whenever the reverser restarts, and leaves the watch resume cursors +inactive unless you configure Redis separately. The chart's `KubeProvider`, `local` in the release namespace, admits only the release namespace by default. Grant the namespaces whose `GitTarget`s may mirror through it in @@ -119,6 +119,11 @@ other subresources. Keep manual scale writes if you want their authors recorded. The example includes Secret writes at `RequestResponse` level. To keep Secret bodies out of the webhook, exclude core `secrets` too. Those writes then have no audit evidence for attribution. + +Keep `create` of `commitrequests` in the `configbutler.ai` group covered, at `Metadata` level at +least: a `CommitRequest`'s author comes from its own audit event, as a write's does. Without it a +request names nobody, and cannot save the window of an author the audit names. The write catch-all +covers it. The audit policy controls evidence collection; `WatchRule` and `ClusterWatchRule` control what gets written to Git. diff --git a/docs/bi-directional-flux.md b/docs/bi-directional-flux.md index e4dc209f5..13efc35a4 100644 --- a/docs/bi-directional-flux.md +++ b/docs/bi-directional-flux.md @@ -582,11 +582,12 @@ applied the tip is asked nothing. | Every operation checks the target again (incarnation, branch, admission, consent) and the source binding before it resumes, records itself first, and never asks for something weaker; a stop mid-apply suspends the `Kustomization` | [`dispatcher_test.go`](../internal/applier/dispatcher_test.go) | | A failed attempt is decided again; each `Kustomization` keeps its own backoff; a shared one is asked once; appliers sharing a source apply one at a time; nothing is sent while no flush can be made; an undo of an edit in the apply window is mirrored, and the applier's own write is not | [`apply_attempt_test.go`](../internal/git/apply_attempt_test.go), [`apply_groups_test.go`](../internal/git/apply_groups_test.go), [`applier_claim_test.go`](../internal/git/applier_claim_test.go), [`apply_window_test.go`](../internal/git/apply_window_test.go) | | A newly named `Kustomization` converges once; the startup converge asks nothing of one at the tip; an API create is adopted even at the tip; nothing is asked before the initial snapshots are published, or for an empty tip | [`apply_catchup_test.go`](../internal/git/apply_catchup_test.go), [`applier_claim_test.go`](../internal/git/applier_claim_test.go) | +| A burst of the worker's own pushes converges once after `spec.catchUpAfter`; `0s` never does; a changed interval applies from the next wake; an interval converge yields to an edit before its resume and waits a fresh interval, where a `ForeignCommit` does not | [`apply_converge_test.go`](../internal/git/apply_converge_test.go) | | An object another applier owns is neither mirrored nor swept | [`apply_ownership_test.go`](../internal/git/apply_ownership_test.go) | | An applier claim nothing names is `Ready` and holds nothing; named from a `GitTarget` on another branch than its source, it is `SourceMismatch` and adopts nothing | [Manager Flux Applier](../test/e2e/flux_applier_e2e_test.go) | | A `ForeignCommit` for a `SourceMismatch` applier never stamps the other branch's source and records `ApplyRequestFailed`; once the source is fixed the applier claim adopts the `Kustomization` and Flux reaches the tip; dropping the reference hands it back, and naming the applier claim again adopts it anew | Manager Flux Applier | | A Gitea push webhook to the operator's receiver gets a Git-side push applied; with no receiver and no API edit, the refresh finds one | Manager Flux Applier | -| A failed apply is suspended again and retried; a `flux resume` is suspended again with a `ClaimedResourceReconciled` Event; deleting the applier claim hands the `Kustomization` back | Manager Flux Applier | +| A failed apply is suspended again and retried; a `flux resume` is suspended again with a `ClaimedResourceReconciled` Event; a mirrored API edit's tip is applied as a `CatchUp` at `spec.catchUpAfter`, so `lastAppliedRevision` and the artifact reach it; deleting the applier claim hands the `Kustomization` back | Manager Flux Applier | | A manager restarted during an apply suspends the `Kustomization` as it stops; the new one completes that same operation and leaves it suspended | [Flux Restart Recovery](../test/e2e/flux_restart_recovery_e2e_test.go) | | Two API edits seconds apart end at the later value with no extra commit | [Bi-directional (Flux)](../test/e2e/flux_bi_directional_e2e_test.go) | | A Git-side push with an API edit on top lands both, and Flux applies the replayed commit | Bi-directional (Flux) | diff --git a/docs/bi-directional.md b/docs/bi-directional.md index 81ec57ba3..7fdafe033 100644 --- a/docs/bi-directional.md +++ b/docs/bi-directional.md @@ -281,6 +281,12 @@ nothing else. `KubeProvider`s, and, for a `Refusal`, still consents; and the applier claim still holds the applier. When any of that changed, it asks nothing and decides again, after a new flush. It never asks for something weaker than what it was told. +- **The apply window waits for the watch.** After the applier reports the apply finished, the window + stays open for a 10-second grace, and until the watch has delivered every write the source cluster + held at that moment, for at most two minutes. Each event waits for its audit fact first, and on a + cluster that sends none, for the whole attribution grace, so the applier's writes can reach the + branch worker long after the apply finished. They are still judged inside the window + (`apply_window_barriers_total`). - **The apply's own writes are not mirrored.** Inside the apply window, a write whose newest field managers are the applier's alone, made since the request was sent, to an object nobody edited in the window, is the apply landing. It is kept out of Git and counted in diff --git a/docs/components.md b/docs/components.md index 3174d6539..ec157ac10 100644 --- a/docs/components.md +++ b/docs/components.md @@ -78,7 +78,7 @@ The operator's own API surface and the reconcilers behind it. |---|---|---| | CRD types | [`api/v1alpha3/`](../api/v1alpha3/) | the ten kinds users apply | | Reconcilers | [`internal/controller/`](../internal/controller/) | one per kind, plus shared condition and status helpers | -| Admission handlers | [`internal/webhook/`](../internal/webhook/) | the always-allow observer and `validate-operator-types`, which captures a `CommitRequest` submitter | +| CommitRequest core | [`internal/controller/commitrequest_core.go`](../internal/controller/commitrequest_core.go) | serves a `CommitRequest` for both paths: the control-cluster reconciler, and the server that watches source clusters' requests through their `KubeProvider`s | | Access grants | [`internal/authz/`](../internal/authz/) | one admission decision per act, read from the grant lists on the referenced object (`allowedGitBranches`, `allowedGitTargets`, `allowedApplierClaims`) with the scope that applies, in their own package because several call sites need the same answer | | Rule store | [`internal/rulestore/`](../internal/rulestore/) | compiled `WatchRule` and `ClusterWatchRule` cache the data plane reads | @@ -137,7 +137,6 @@ Optional, and deliberately unable to affect *what* is written. It only names the | Fact transport | [`internal/queue/fact_stream.go`](../internal/queue/fact_stream.go) | Redis Streams by default, an in-process ring with `--author-attribution-transport=memory` (which frees attribution from Redis, but not the watch resume cursors) | | Fact index and follower | [`internal/queue/fact_index.go`](../internal/queue/fact_index.go) | a bounded, TTL'd in-process index, followed per type while at least one watch needs it | | Author resolver | [`internal/watch/author_resolver.go`](../internal/watch/author_resolver.go) | joins a watch event to a fact within a bounded grace window | -| Command author store | [`internal/queue/command_author_store.go`](../internal/queue/command_author_store.go) | the `CommitRequest` submitter captured at admission | ### Support and tooling @@ -189,7 +188,9 @@ flowchart LR | Target watches | **objects** of claimed types | source cluster | raw `dynamic ... Watch()`, no informer and no cache ([`internal/watch/target_watch.go:1420`](../internal/watch/target_watch.go#L1420)) | Two more paths reach the operator without being watches at all. The audit webhook is a **push** from -kube-apiserver, and the admission webhooks are synchronous requests. Neither observes state. The +kube-apiserver; it observes no state. The `CommitRequest` server's watch of a source cluster's +requests is a raw `dynamic ... Watch()` too, one per `KubeProvider` that reconciles them, scoped to +the namespaces it lends. The applier claim reconciler reads each applier claim's `Kustomization` or `Application` with a GET on its own tick rather than a watch, so a cluster without Flux or Argo CD needs no informer for them. diff --git a/docs/configuration.md b/docs/configuration.md index 821f15921..2749da5b2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -105,6 +105,7 @@ Set exactly one of `secretRef` and `inCluster`. Both are immutable: delete and r | `client.qps` | `20` | Client rate limit for this source cluster. Refused beside `inCluster` | | `client.burst` | `30` | Client burst for this source cluster. Refused beside `inCluster` | | `attribution.auditRoute` | `.` | Audit route this provider joins. A route is held by one namespace. See [audit route](#audit-route-and-auditfactsreceived) | +| `commitRequests` | `Ignore` | Whether the `CommitRequest`s created on this cluster are served, and finished ones deleted: `Ignore`, `Reconcile`, `ReconcileAndDeleteFinished`. See [serving them](#serving-the-commitrequests-of-a-source-cluster) | ### `GitTarget` (namespaced): which branch and folder @@ -118,6 +119,7 @@ The destination fields are immutable: to move a target, delete it and create a n | `commit.window.idleTimeout` | `5s` | Close a commit window after this much silence. `0s` commits every write on its own. See [the commit window](#the-commit-window-speccommitwindow) | | `commit.window.maxDuration` | `1m` | Close a commit window this long after it opened, however much keeps arriving | | `commit.message` | the built-in templates | How commits are phrased. See [commit messages](commit-messages.md) | +| `allowedCommitRequests` | `[{"consumerNamespaces":{"from":"Same"}}]` | The `CommitRequest`s that may save this target's windows: `consumerNamespaces` on the cluster GitOps Reverser runs in, `sourceNamespaces` on the one it mirrors. See [who may save](#who-may-save-specallowedcommitrequests) | | `placement.byType` | the built-in path | Per-type path templates for new documents. See [where new resources are written](#where-new-resources-are-written-specplacement) | | `placement.default` | the built-in path | Catch-all path template for types with no `byType` entry | | `placement.useKustomize` | off | Keep the folder a kustomize folder. See [keeping the folder a kustomize folder](#keeping-the-folder-a-kustomize-folder-specplacementusekustomize) | @@ -159,6 +161,7 @@ The references are immutable: to hold another `Kustomization`, delete the applie |---|---|---| | `kubeProviderRef` | **required** | The `KubeProvider` whose cluster holds the `Kustomization`: the applier claim's own namespace, or `namespace` names another. Its `allowedApplierClaims` must admit this namespace | | `kustomizationRef` | **required** | The `Kustomization` that applies the folders of the `GitTarget`s naming this applier claim. See [who applies this folder](#who-applies-this-folder-specapplierclaimref) | +| `catchUpAfter` | `1m` | How long the `Kustomization` stays behind the branch after a push of GitOps Reverser's own before an apply of the tip is asked for. Such a push needs no apply for the cluster, but leaves `status.lastAppliedRevision` and the `GitRepository`'s artifact at the commit before it. The interval starts when the `Kustomization` is first seen behind, so a burst of saves costs one apply; the apply can still wait for startup, a retry backoff or edits being published, and an edit arriving before the resume defers it by an interval. `0s` never asks for this reason, and drops an apply it asked for that has not started; otherwise `10s` to `24h` | ### `ArgoCDClaim` (namespaced): the Argo CD `Application` GitOps Reverser asks to sync @@ -168,18 +171,20 @@ The references are immutable: to hold another `Application`, delete the applier |---|---|---| | `kubeProviderRef` | **required** | The `KubeProvider` whose cluster holds the `Application`: the applier claim's own namespace, or `namespace` names another. Its `allowedApplierClaims` must admit this namespace | | `applicationRef` | **required** | The `Application`, with automated sync off, that syncs the folders of the `GitTarget`s naming this applier claim. See [who applies this folder](#who-applies-this-folder-specapplierclaimref) | +| `catchUpAfter` | `0s` | As a `FluxClaim`'s: how long the `Application`'s last sync stays behind the branch after a push of GitOps Reverser's own before a sync of the tip is asked for. Off by default, since a sync runs the `Application`'s hooks and Argo CD's refresh already compares the live state against the tip. Otherwise `10s` to `24h` | -### `CommitRequest` (namespaced): save now +### `CommitRequest` (namespaced, here or on a source cluster): save now | Field | Default | What it does | |---|---|---| -| `gitTargetRef` | **required** | The target whose commit window the request attaches to | +| `gitTargetRef` | **required** | The target whose commit window the request attaches to: in this namespace, or `namespace` names another, whose `allowedCommitRequests` must admit this one. On a source cluster `namespace` is required. See [where a request is created](#where-a-request-is-created) | | `message` | the target's templates | Commit message, committed verbatim unless `requestTemplate` frames it | | `window.attach` | `CurrentOrNext` | Which window to attach to: the author's current or next one, or only the next. See [the request window](#the-request-window-specwindow) | -| `window.attachTimeout` | `2s` | How long to wait for a window to attach to | +| `window.attachTimeout` | `5s` | How long to wait for a window to attach to | | `window.idleTimeout` | no idle close | Close the attached window after this much silence | | `window.maxDuration` | `2s` | Close the attached window this long after the attach | | `whenNothingToCommit` | `Resolve` | `CommitEmpty` records the message in an empty commit when nothing changed | +| `catchUp` | `AsDefinedOnClaim` | `AfterPush` asks the `GitTarget`'s applier to apply the branch tip right after the commit is pushed, instead of after the applier claim's `catchUpAfter`. The tip may hold later commits, and saves pushed close together can share one apply; the `Applied` condition reports it | @@ -1892,10 +1897,11 @@ for a `Kustomization` and an `Application`. | `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. The pending writes are replayed onto it and pushed first, so the request names that pushed commit | Every applier declared on the branch, once per applier | | `Refusal` | An eligible refused edit, with `spec.onRefusal: Reapply` | This target's applier | | `Repair` | An apply reverted an API edit made after the flush; Git kept the edit | That applier | -| `CatchUp` | To adopt: the API created an object in the folder, or a resync added an object to Git. To converge: once when the operator starts, and once when a `GitTarget` first names the applier claim | To adopt, that folder's applier; to converge, every applier declared on the branch at start, and the newly named one | +| `CatchUp` | To adopt: the API created an object in the folder, or a resync added an object to Git. To converge: once when the operator starts, once when a `GitTarget` first names the applier claim, once the applier claim's `catchUpAfter` has passed since a push of GitOps Reverser's own, and after a `CommitRequest` with [`catchUp: AfterPush`](#applying-a-save-at-once) pushed a commit | To adopt, that folder's applier; to converge, every applier declared on the branch at start, the newly named one, the one left behind, or the requesting `GitTarget`'s | -An API edit asks for no apply, and neither does a push of GitOps Reverser's own: the cluster already -runs what they hold. A create is different, because an applier prunes only what it applied: an +An API edit asks for no apply, and neither does a push of GitOps Reverser's own, at once: the cluster +already runs what they hold. Only the applier's own state is left behind, which `catchUpAfter` +catches up later. A create is different, because an applier prunes only what it applied: an object the API created (an `ADDED` event whose newest field managers are not an applier's) carries neither Flux's owner labels nor Argo CD's tracking annotation, so a later Git-side delete of its file would leave it running. An adopt is therefore applied even at the commit the applier last @@ -2708,6 +2714,13 @@ The important fields are: - `spec.rules[].sourceNamespace`: the source-cluster namespace that item watches; omitted means the rule's own namespace +**The `kube-root-ca.crt` ConfigMap is never mirrored**, whatever a rule selects. The cluster writes +it into every namespace, it holds that one cluster's CA, and the cluster overwrites any other value, +so a copy in Git can be neither edited nor restored. A copy already in Git is treated like any +document the cluster does not hold: prune mode `Always` removes it on the next resync, and under +`OnEvent` you delete it yourself. Each snapshot that leaves it out logs `left out cluster-generated +objects on purpose`, and `watch_events_total{outcome="cluster_generated"}` counts its live events. + ### Watching a different source namespace Set `spec.rules[].sourceNamespace` to mirror a namespace other than the one the `WatchRule` lives in @@ -2939,18 +2952,26 @@ Use it sparingly. ## `CommitRequest` -`CommitRequest` is a one-shot "save" signal for a same-namespace `GitTarget`. It does not create or -change watch rules. It attaches to its author's commit window, gives that window its message, and -decides when the window closes, instead of the target's timers. +`CommitRequest` is a one-shot "save" signal for a `GitTarget`. It does not create or change watch +rules. It attaches to its author's commit window, gives that window its message, and decides when +the window closes, instead of the target's timers. + +It is the one kind of GitOps Reverser that can also live on a source cluster. A request for a +`GitTarget` that mirrors the cluster GitOps Reverser runs in is created there; a request for a +`GitTarget` that mirrors another cluster is created on that cluster, next to the edits it saves. See +[where a request is created](#where-a-request-is-created). The important fields are: -- `spec.gitTargetRef.name`: the target whose window to attach to +- `spec.gitTargetRef`: the target whose window to attach to, by `name`, in the request's own + namespace unless `namespace` names another; see [who may save](#who-may-save-specallowedcommitrequests) - `spec.message`: optional literal commit message, preserved verbatim - `spec.window`: which window to attach to and when it closes; see [the request window](#the-request-window-specwindow) - `spec.whenNothingToCommit`: what to do when the request ends with nothing to commit; see [when there is nothing to commit](#when-there-is-nothing-to-commit) +- `spec.catchUp`: whether the applier is asked to apply the save at once; see + [applying a save at once](#applying-a-save-at-once) Example: @@ -2969,7 +2990,7 @@ spec: Route traffic to the port exposed by the API container. window: attach: CurrentOrNext - attachTimeout: "2s" + attachTimeout: "5s" maxDuration: "2s" ``` @@ -2983,6 +3004,30 @@ configure [`requestTemplate`](commit-messages.md#framing-a-save-message) on the parsed as a template. A rejected request leaves automatic mirroring available. The submitter chooses any semantic prefix; free-form messages are accepted. +### Where a request is created + +The `GitTarget` a request names decides where it belongs, through the `KubeProvider` its +`spec.kubeProviderRef` names: + +| The `GitTarget`'s `KubeProvider` | Create the request on | `gitTargetRef.namespace` | +| --- | --- | --- | +| `inCluster` | the cluster GitOps Reverser runs in | optional: omitted, the request's own namespace | +| a kubeconfig, with `spec.commitRequests` set to a `Reconcile` value | the cluster that kubeconfig connects to | required: a namespace of the cluster GitOps Reverser runs in | + +- **A request on a source cluster must name the `GitTarget`'s namespace.** That namespace belongs to + the cluster GitOps Reverser runs in, so there is nothing on the source cluster to default to. A + request without it is never picked up there, and gets no status. +- **A request on the cluster GitOps Reverser runs in, for a `GitTarget` that mirrors another + cluster, is refused** with reason `NotAllowed`, when that `KubeProvider` serves no + `CommitRequest`s: its author would come from this cluster's audit and its edits' authors from the + other one's. Create it on the other cluster instead. When that `KubeProvider` does serve them, the + request is left alone, since the kubeconfig may point back at this cluster and then its own watch + serves the request. If it points elsewhere, nothing serves the request and it gets no status. + +Only the path the `GitTarget`'s `KubeProvider` picks writes a request's progress, so several +`KubeProvider`s that watch one cluster never fight over a request. Any of them may report +`GitTargetNotFound` for a request whose `GitTarget` does not exist once `attachTimeout` has passed. + ### The request window (`spec.window`) A request never opens a window; only writes do. It waits for one, attaches to at most one, and @@ -2991,7 +3036,7 @@ replaces that window's timers with its own, shorter or longer: | Field | Meaning | Default | |---|---|---| | `attach` | `CurrentOrNext` attaches to the author's window if one is open when the worker registers the request, and otherwise to the next one a write opens. `Next` first closes the author's open window, under the message it already carries, and attaches to the next one | `CurrentOrNext` | -| `attachTimeout` | stop waiting for a window this long after the worker registers the request | `2s` | +| `attachTimeout` | stop waiting for a window this long after the worker registers the request | `5s` | | `idleTimeout` | close the attached window after this much silence; every write restarts it | no idle close | | `maxDuration` | close the attached window this long after the attach, however much keeps arriving | `2s` | @@ -3020,12 +3065,12 @@ and restarts nothing: its deadlines and, for `Next`, its one close are fixed at #### Sizing the request window -`attachTimeout` covers the gap between creating the request and its first write reaching the +`attachTimeout` counts from when GitOps Reverser picks the request up, which is after the request's +own audit event arrived. It covers the gap between that and the request's first write reaching the branch worker. A write is held in the watch path until its audit fact arrives, so the floor is the -API server's `--audit-webhook-batch-max-wait` plus the attribution join (roughly 1 to 1.5 seconds at -the reference configuration of `1s`). The default of `"2s"` clears that; a loaded or distant cluster -may want `"4s"` to `"5s"`, and a save created before its writes needs the time to its first write -on top. +spread between the request's audit event and its writes'. The default of `"5s"` clears that on a +loaded or distant cluster; a save created before its writes needs the time to its first write on +top. The floor assumes the stream is not busy. A watch stream attributes its events in order, so a save's edit can also queue behind an earlier event of the same type that is still waiting for its fact, for @@ -3033,26 +3078,109 @@ up to `--author-attribution-grace` each. That is rare when the audit policy cove [suggested alert](interpreting-metrics.md#suggested-alerts) on `tier="absent"` says when it is not. Widening `attachTimeout` hides that delay rather than removing it. +A request is registered with the branch worker only once its own audit event has named its author, +and its edit is attributed the same way, so a request created just before its edit does not always +reach the worker first. On a `GitTarget` whose windows close on the first silence (`idleTimeout: +"0s"`), such an edit can commit alone before the request is there to attach to it. Make the edit +once the request is claimed (`Ready=Unknown`), or give the `GitTarget` an `idleTimeout` that covers +the gap. + `maxDuration`, and `idleTimeout` when set, cover the span of the save's own writes after the attach. The worst case from registration to the finalize is `attachTimeout` plus `maxDuration`; the push follows, with its cooldown and any retries. -Do not size either against `--author-attribution-grace`. When that grace expires with no fact the -write still ships, as a window that names no actor, and a request naming a submitter can never -attach to it: the outcome is `WindowMismatch` no matter how long the request waits. - The two directions are not symmetric. Overshooting costs a few seconds of latency on the commit; undershooting `attachTimeout` resolves the request `Ready=True` with reason `NoWindow` while the edit commits seconds later under the target's `liveTemplate`, which reads as a save button that did nothing. A request cannot rename a finalized commit, including a local commit waiting for push. Between the -finalize and the push the request is still in flight, and the controller keeps re-sending its +finalize and the push the request is still in flight, and GitOps Reverser keeps re-sending its attach until it reads an outcome; the worker recognizes the re-send as the same request, so a request never resolves `NoWindow` while its commit waits out the push cooldown. Applying resources and a request together gives no ordering guarantee between controllers and watch streams. See the [request contract](spec/commitrequest-design.md). +### Who may save: `spec.allowedCommitRequests` + +A `GitTarget` says which `CommitRequest`s may save its windows in `spec.allowedCommitRequests`, a +grant list like the others (see [the grants](#the-grants)). Each grant sets exactly one of two +fields, one for each place a request can be created: + +- **`consumerNamespaces`** admits requests on the cluster GitOps Reverser runs in, by namespace, + with `from: Same | Names | Selector | All`, as in every other grant list. It admits requests only + for a `GitTarget` whose `KubeProvider` sets `inCluster`. +- **`sourceNamespaces`** admits requests on the cluster the `GitTarget` mirrors, by exact namespace + name, or `"*"` for every namespace the `GitTarget` may mirror. Variables are refused. A namespace + the `KubeProvider` does not lend this `GitTarget` in `spec.allowedGitTargets` is never admitted, + the same bound its `WatchRule`s have. It admits requests only for a `GitTarget` whose + `KubeProvider` names a kubeconfig. + +The list follows the pattern of the other grant lists: + +- **Omitted:** the `CommitRequest`s in the `GitTarget`'s own namespace, as one grant with + `consumerNamespaces: {from: Same}`. +- **`[]`:** no `CommitRequest`. +- **A list:** the `CommitRequest`s at least one grant admits. The list replaces the default, so a + `GitTarget` that mirrors another cluster writes a `sourceNamespaces` grant. + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +metadata: + name: shared + namespace: platform +spec: + allowedCommitRequests: + - consumerNamespaces: {from: Same} + - consumerNamespaces: + from: Selector + selector: + matchLabels: + platform.example.com/tenant: "true" +``` + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +metadata: + name: shop + namespace: team-shop +spec: + kubeProviderRef: {name: shop-workspace} + allowedCommitRequests: + - sourceNamespaces: ["*"] +``` + +A request no grant admits fails before it attaches: `Ready=False`, `Stalled=True`, reason +`NotAllowed`. The message says which field applies on the cluster the request was made on. + +### Who the save is from + +A request's author is found from its own audit event, on the cluster it was created on, the same +way an edit's author is: from the same index, with the same `--author-attribution-grace`. So a save +attaches only to its creator's windows, by the ordinary author rule, and `status.author` names the +creator. The author is recorded in the request's first status write, before it attaches. Create a +request on a source cluster as JSON: `kubectl` and browsers send JSON, and an API server that drops +the audit event of a YAML body loses the save's author without an error. + +| The cluster the request was made on | The save | +| --- | --- | +| sends audit events, and one names the creator | attaches to the creator's windows; `status.author` names them | +| sends no audit events at all (its `KubeProvider` never received one) | names nobody, and attaches to a window whose edits name nobody; the commit's author stays unresolved | +| sends audit events, but none names the creator | is refused, `Ready=True` with reason `AuthorUnresolved`: its event went missing, and attaching it would claim the window that should stay unattributed. Save again | +| any, with author attribution off | names nobody, and attaches to the `GitTarget`'s one unnamed window; the committer authors the commit | + +Who may save is decided by `allowedCommitRequests`; who authored the commit is decided by audit, or +by nobody. + +### What a save holds + +A commit window holds one author's edits to one `GitTarget`, wherever that `GitTarget` mirrors them. +An admitted request therefore saves its creator's whole window, edits outside the request's own +namespace included: a grant in `allowedCommitRequests` authorizes exactly that. The request's +namespace decides only whether a grant admits it. + ### When there is nothing to commit `spec.whenNothingToCommit` decides what a request does when it ends with nothing to commit: its @@ -3069,6 +3197,7 @@ reason, without changing Git. | The writes already matched Git | no commit, `AlreadyPresent` | empty commit, `AlreadyPresent` | | No eligible window before `attachTimeout` | no commit, `NoWindow` | empty commit, `NoWindow` | | Only another author's window was open | no commit, `WindowMismatch` | no commit, `WindowMismatch` | +| An earlier save of the same author on the `GitTarget` was still in progress | `SaveInProgress`, at once | `SaveInProgress`, at once | | The target is suspended, or its render fidelity is not established | `FinalizeFailed`, the cause | `FinalizeFailed`, the cause | | The commit or the empty commit failed | `FinalizeFailed` | `FinalizeFailed` | | The push failed | `WaitingForPush`, with the cause, until a push lands; `FinalizeFailed` if the worker stops first | the same | @@ -3081,18 +3210,32 @@ author's window never records one, because the history would then claim more tha A crash between the push and the status write can record the message twice: the worker recognizes a repeated request only while it runs. +### Its status + Progress and outcome are reported through kstatus-compatible **conditions** (no `phase` string). -`kubectl get commitrequest` shows `Ready`, its `Reason` and the `Commit`; `-o wide` adds -`AuthorAttributed` and `Pushed`. Automation must stop on either `Ready=True` or `Stalled=True`; +`kubectl get commitrequest` shows `Ready`, its `Reason`, the `Author` and the `Commit`; `-o wide` +adds `Pushed` and `Applied`. Automation must stop on either `Ready=True` or `Stalled=True`; `kubectl wait --for=condition=Ready` alone keeps waiting on terminal failures. `Ready=True` includes successful no-commit outcomes. Require `Pushed=True` and `status.commit` for evidence that the request produced a pushed commit: -- **Ready** (summary): `True` once the request reached a non-error terminal outcome. The `Ready` - condition's `reason` says which: `Committed` (a commit was pushed; `status.branch`/`status.commit` set), - or an outcome with nothing to commit: `NoWindow`, `WindowMismatch`, or `AlreadyPresent`. With - [`CommitEmpty`](#when-there-is-nothing-to-commit), `NoWindow` and `AlreadyPresent` can come with a - recorded empty commit. A failed finalize is `Ready=False` with reason `FinalizeFailed`. +- **Ready** (summary): `Unknown` while the save runs, from the first status write, which records the + author. `True` once the request reached a non-error terminal outcome. The `Ready` condition's + `reason` says which: `Committed` (a commit was pushed; `status.branch`/`status.commit` set), or an + outcome with nothing to commit: `NoWindow`, `WindowMismatch`, `AlreadyPresent`, or + `AuthorUnresolved`. With [`CommitEmpty`](#when-there-is-nothing-to-commit), `NoWindow` and + `AlreadyPresent` can come with a recorded empty commit. `Ready=False` with `Stalled=True` is a + failure: `NotAllowed` when the `GitTarget` does not admit the request, or the request was made on + the wrong cluster; `GitTargetNotFound` when the `GitTarget` did not exist within `attachTimeout`; + `SaveInProgress`; or `FinalizeFailed`. + + `SaveInProgress` means an earlier request of the same author on the same `GitTarget` was still + waiting for a commit window, or collecting one, when this one arrived. A window carries one + message, so the two would race for it and one message would be lost without a word; the later + request is refused at once instead. Save again with a new request once the earlier one is `Ready`. + A request for the `Next` window is not refused by one that is collecting: it closes that window + under the earlier request's message first. Saves by different authors never race, since each + author has windows of its own. `AlreadyPresent` means **the remote confirmed there was nothing to add**, so it is reported after the push rather than when the local plan found no difference. That costs a few seconds and buys a @@ -3100,22 +3243,44 @@ request produced a pushed commit: the worker replays onto the moved branch, and such a request resolves `Committed` instead. A worker that stops before the push fails the request rather than leaving it to time out. - **Reconciling** / **Stalled**: the kstatus progress/blocked pair. `Reconciling=True` while the - request is in progress, with the phase the branch worker reports as its reason: `Progressing` - (sent, not yet registered), `WaitingForWorker` (the GitTarget has no branch worker yet), + request is in progress, with the phase the branch worker reports as its reason: `Saving` + (claimed, not yet registered), `WaitingForWorker` (the GitTarget has no branch worker yet), `WaitingForWindow`, `CollectingWindow`, or `WaitingForPush`. - `Stalled=True` when the finalize failed and needs attention (kstatus reports the object Failed). -- **AuthorAttributed**: `True` with reason `AttributedFromAdmission` when the internal commands - admission webhook captured the request submitter. `False` with reason `CommitterFallback` means capture - ran but no admission record exists; `False` with reason `AuthorCaptureDisabled` means capture is not - configured. Neither is a failure. The request then claims no actor and can attach only to an unnamed - watch window, whose Git author remains either the configured committer or the explicit unresolved author - according to the watch attribution outcome. + `Stalled=True` when the request failed and needs attention (kstatus reports the object Failed). - **Pushed**: `True` once the commit, empty or not, is in the remote repository. +- **Applied**: only with [`catchUp: AfterPush`](#applying-a-save-at-once), once `Ready=True`: + whether the applier caught up with the commit. It never changes `Ready`, so `Ready=True` alone + does not mean the commit was applied. + +A request is served again after GitOps Reverser restarts: the path that owns it finds its own +claim, and resumes with the author it recorded. + +### Applying a save at once + +A save is in Git once it is pushed, and the cluster already runs it, so by default nothing applies +it: the applier claim's `catchUpAfter` brings the applier's own state (a `Kustomization`'s +`status.lastAppliedRevision`, its `GitRepository`'s artifact) level later. A request that needs that +state at once, to publish what was applied, sets `spec.catchUp: AfterPush`. Its +`GitTarget`'s applier is then asked to apply the branch tip right after the commit is pushed. The +tip may hold later commits, saves pushed close together can share one apply, and an applier +already at the tip needs none. The request reports the outcome in its `Applied` condition: + +| `Applied` | Reason | Means | +| --- | --- | --- | +| `Unknown` | `Applying` | the apply is asked for or runs | +| `Unknown` | `Retrying` | it did not complete and is asked again; the message says why | +| `True` | `Applied` | the applier applied a revision that holds the commit; `status.appliedRevision` names it, as the applier reported it | +| `True` | `NothingToApply`, `NoApplierClaim`, `ApplierGone` | no apply was needed: the request pushed no commit, or the `GitTarget` names no applier claim, or stopped naming the one asked. The cluster already holds what was saved | +| `False` | `CommitDropped` | the branch no longer holds the commit: its history was rewritten after the push. The only `False` | +| `Unknown` | `Lost` | GitOps Reverser restarted before it learnt how the apply went | + +Wait for `Applied=True` for "saved and applied"; `Ready=True` stays "saved". A finished request is +deleted only after its `Applied` condition is settled. ### Finished requests are deleted after 48 hours Every save leaves one `CommitRequest` behind. When a request finishes (`Ready=True` or -`Stalled=True`), the controller writes the time it will be deleted onto it: +`Stalled=True`), GitOps Reverser writes the time it will be deleted onto it: ```yaml metadata: @@ -3123,13 +3288,13 @@ metadata: configbutler.ai/delete-after: "2026-10-01T09:30:00Z" ``` -Once that time has passed, the controller deletes the request. The annotation is the whole -contract, so it is also how you change the outcome for one request: +Once that time has passed, the request is deleted. The annotation is the whole contract, so it is +also how you change the outcome for one request: - Remove the annotation to keep the request. - Edit the time, in RFC 3339 format, to delete it earlier or later. A value that does not parse keeps the request. -- Set it yourself when you create the request to choose the time up front. The controller never +- Set it yourself when you create the request to choose the time up front. GitOps Reverser never replaces a value that is already there. The period comes from `--commit-request-ttl` (Helm: `controllerManager.commitRequestTTL`, default @@ -3137,6 +3302,69 @@ The period comes from `--commit-request-ttl` (Helm: `controllerManager.commitReq requests are kept. A request that finished before this feature existed has no annotation and is never deleted. +On the cluster GitOps Reverser runs in this always applies. On a source cluster it applies only +when the `KubeProvider` sets `spec.commitRequests: ReconcileAndDeleteFinished`; with `Reconcile`, +nothing is stamped and nothing is deleted, and annotations stamped before a switch back to +`Reconcile` stay without effect. + +### Serving the `CommitRequest`s of a source cluster + +Three things turn it on, each on purpose: + +1. **Install the CRD on the source cluster.** It is one CRD, the release asset + `commitrequest-crd.yaml`, apart from the other `configbutler.ai` kinds, which a source cluster + does not need. Install it with `kubectl apply --server-side -f commitrequest-crd.yaml`. GitOps + Reverser never creates or updates it there. +2. **Set `spec.commitRequests` on the `KubeProvider`.** GitOps Reverser then watches the cluster's + `CommitRequest`s through that provider, in the namespaces it lends its `GitTarget`s in + `spec.allowedGitTargets` (one watch across the cluster when a grant lends `"*"`), and writes each + one's status back through the same credential. A `CommitRequest` is never mirrored, whatever a + `WatchRule` selects. +3. **Grant it on the `GitTarget`, in `spec.allowedCommitRequests`,** with `sourceNamespaces`. The + provider switch makes the requests reachable; the `GitTarget`'s owner decides which of them may + save its windows. + +Each value of `spec.commitRequests` makes the cluster a bigger source of `CommitRequest`s, and needs +more of the credential on `commitrequests` in the `configbutler.ai` group, in each lent namespace: + +| Value | What GitOps Reverser does | The credential needs | +| --- | --- | --- | +| `Ignore` (default) | nothing | nothing | +| `Reconcile` | serves the requests in the namespaces this `KubeProvider` lends its `GitTarget`s, and writes their status. A finished request stays, and no `delete-after` is stamped on it, because nothing would act on it | `get`, `list`, `watch`; `update` on `commitrequests/status` | +| `ReconcileAndDeleteFinished` | as `Reconcile`, and also stamps `configbutler.ai/delete-after` when a request finishes, and deletes the request once that time has passed | as `Reconcile`, plus `patch` and `delete` | + +A `KubeProvider` with `inCluster` takes only `Ignore`: the `CommitRequest`s of the cluster GitOps +Reverser runs in are always served, with the operator's own identity. + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: KubeProvider +metadata: + name: shop-workspace + namespace: team-shop +spec: + secretRef: {name: shop-workspace} + commitRequests: ReconcileAndDeleteFinished +``` + +The `KubeProvider` reports how serving goes in its `CommitRequestsServed` condition, which is not +part of `Ready`, and the namespaces it watches in `status.commitRequests.namespaces`. Both are absent +while `spec.commitRequests` is `Ignore`. `kubectl get kubeprovider -o wide` shows the reason in the +`Saves` column: + +| Status | Reason | When | +| --- | --- | --- | +| `True` | `Watching` | the message says how many namespaces, or "every namespace" | +| `True` | `NothingToWatch` | no `GitTarget` mirroring through this `KubeProvider` is lent a namespace | +| `False` | `CRDNotInstalled` | the cluster serves no `commitrequests.configbutler.ai`; apply `commitrequest-crd.yaml` | +| `False` | `VersionNotServed` | the CRD is there, but does not serve `v1alpha3`; apply `commitrequest-crd.yaml` of this release | +| `False` | `Forbidden` | `list` or `watch` was denied; the message names the namespaces and the verbs to grant | +| `Unknown` | `Unreachable` | the cluster's API could not be read; the message carries the error | + +It is checked on every resync, so a CRD applied by hand clears it without a restart. A finished +request the credential may not delete stays in place; the message says so, and the condition stays +`True`, because serving works. + ## Audit ingestion settings Object state comes from Kubernetes **watch**, not from audit. Audit is an optional attribution lookup: diff --git a/docs/definitions.md b/docs/definitions.md index 404448726..aac5f0f21 100644 --- a/docs/definitions.md +++ b/docs/definitions.md @@ -42,7 +42,8 @@ without a side, or "namespace" without a direction. Fix the side first and the r vocabulary follows. **Control cluster.** The cluster the operator runs in, and where its own ten kinds live. When a -document says "a namespace" with no qualifier, it means a control-cluster namespace. +document says "a namespace" with no qualifier, it means a control-cluster namespace. `CommitRequest` +is the one operator kind that can also live on a source cluster, next to the edits it saves. **Source cluster.** The cluster whose objects are mirrored into Git. A `GitTarget` names it through its `KubeProvider`. It is the same cluster as the control cluster on a single-cluster install, @@ -179,7 +180,8 @@ its `maxDuration` after opening, whichever comes first, or earlier for a flush, `CommitRequest` holds it. Windows do not merge across authors, which is why attribution can be per-commit. -**Attach.** What a `CommitRequest` does to a commit window: it gives the window its message and +**Attach.** What a `CommitRequest`, on the control cluster or on a source cluster, does to a commit +window: it gives the window its message and replaces the window's timers with its own. A request attaches to at most one window and never opens one. One word for the act, in the API (`window.attach`, `attachTimeout`), the code and the docs: not "claim", which names a rule's demand on the read side. diff --git a/docs/design/apply-trigger.md b/docs/design/apply-trigger.md index 6d6635d5d..4cdf32ca2 100644 --- a/docs/design/apply-trigger.md +++ b/docs/design/apply-trigger.md @@ -57,7 +57,7 @@ overwrites it is withheld and repaired. Four reasons make one due: | `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. The flush replays the pending writes onto it and pushes them first, so the applied commit carries the API side's answer | Every applier on the branch | | `Refusal` | An edit to a document the folder holds was refused at the write boundary, and the target opted in with `onRefusal: Reapply` and is not suspended. At most one request a minute per target; a refusal inside that minute is carried by a trailing one | The refusing target's applier | | `Repair` | The apply window withheld a late apply over a newer API edit, so the cluster runs an older value than Git | The applier that wrote it | -| `CatchUp` | Ownership, not safety ([below](#catchup-adopt-and-converge)). An **adopt**: the API created an object in the folder, or a resync added one to Git, and an apply stamps it as the applier's, so a later Git-side delete prunes it. A **converge**: once when the worker starts, and once when a `GitTarget` first names the trigger, unless the applier already applied the tip | The folder's applier for an adopt; for a converge, every applier on the branch at start, and the named one on first naming. Flux and Argo CD alike | +| `CatchUp` | Ownership, not safety ([below](#catchup-adopt-and-converge)). An **adopt**: the API created an object in the folder, or a resync added one to Git, and an apply stamps it as the applier's, so a later Git-side delete prunes it. A **converge**: once when the worker starts, once when a `GitTarget` first names the trigger, and once the applier has stayed behind the tip for the claim's `spec.catchUpAfter`, unless the applier already applied the tip | The folder's applier for an adopt; for a converge, every applier on the branch at start, and the named one on first naming. Flux and Argo CD alike | No applier is asked for anything before every initial snapshot of the `GitTarget`s it covers is published, and a push of GitOps Reverser's own leaves nothing pending: the cluster already runs the @@ -259,6 +259,35 @@ deletes its file the applier leaves it on the cluster, and a later resync writes uid. The worker keeps, per applier, the commit the last succeeded operation applied, from the operation's report and, after a restart, from the trigger's `status.operation`. A converge of that commit asks nothing. +- **Converge on an interval.** A push of the worker's own asks nothing at once, but it leaves the + applier's own state behind the branch: a `Kustomization`'s `status.lastAppliedRevision` and its + `GitRepository`'s artifact, an `Application`'s last sync, and whatever reads them (Flux's + commit-status notifications, a step that publishes what was applied). So once an applier has + stayed behind the tip for its claim's `spec.catchUpAfter`, a converge is pending for it. The + clock starts on the first pass that finds it behind and again on every send, so a burst of saves + inside one interval costs one apply, and an applier at the tip costs nothing. A `FluxClaim` + defaults to `1m`; an `ArgoCDClaim` to `0s`, never, since a sync runs the `Application`'s hooks + and Argo CD's refresh already compares the live state against the tip. Setting `0s` also drops + an interval converge still pending, behind the barrier or a backoff; a converge something else + asked for stays. +- **An interval converge yields to an edit.** Its fetch writes nothing to the cluster; its apply + puts the fetched tip there, and an edit mirrored after the send is not in that tip, so the apply + would revert it and a `Repair` would put it back. Such a converge's decision stops holding once a + live edit is mirrored after its send (`ApplyStale`, asked just before the resume), and it waits a + fresh interval rather than going again after the next flush. While a burst of edits lasts, it + lands in the first quiet moment. Every other reason applies regardless, as the flush guarantees + ([above](#flush-then-apply)). +- **A save can ask to catch up at once.** A `CommitRequest` with `catchUp: AfterPush` + leaves a converge pending for its `GitTarget`'s applier as soon as its commit is pushed, without + waiting for `catchUpAfter`. It was asked for, so it does not yield to an edit. A save that pushed + no commit asks nothing. The save waits on its applier, with its commit, until an attempt that + carries it succeeds, and is settled applied only when the applied revision holds that commit: a + force-push that dropped it settles it not applied, and a revision the checkout does not hold yet + keeps it waiting while the remote is looked at. The request's `Applied` condition reports it: + retried with the applier's backoff, or settled applied, also when no apply was needed (nothing + was pushed, or no applier is named: the cluster already holds the save). `False` is kept for + something wrong, a dropped commit, so a client never sees it for a save that went fine + (`save_apply.go`). - **No apply before every initial snapshot is published.** An applier is asked for nothing, for any reason, until every collection of every `GitTarget` it covers has a settled snapshot (applied, refused or failed; tracked per collection on the watch side) and the worker has published every diff --git a/docs/design/branch-worker-event-model.md b/docs/design/branch-worker-event-model.md index b9dc44ea6..9fdd013f8 100644 --- a/docs/design/branch-worker-event-model.md +++ b/docs/design/branch-worker-event-model.md @@ -260,7 +260,7 @@ collect entries older than 15 minutes when no attach is queued. Collection is no durable receipt proving that status was written. Authorship has a separate lifetime. The -[admission author store](../../internal/queue/command_author_store.go) retains records for an hour; +admission author store (since removed) retained records for an hour; the `CommitRequest` object cannot reconstruct the authenticated submitter by itself. Recovering a previously accepted save must preserve its resolved attribution, including an unresolved result. Re-running the lookup later can produce a different decision. @@ -315,7 +315,7 @@ Represent intent, decisions, and external work separately so recovery knows what | Category | Example | Meaning | |---|---|---| -| Command | `SaveRequested` | A caller asks to attach a save | +| Command | `CommitRequested` | A caller asks to attach a save | | Fact | `SaveAttached` | The worker bound that request to a particular window | | Effect intent | `PublishBatch` | A recorded decision that a push must still be attempted | | Effect result | `PublicationConfirmed` | Evidence that a particular attempt published its work | @@ -380,7 +380,7 @@ The current attach loop and controller already preserve several rules: Retain those rules in a durable lifecycle. One illustrative successful path is: ```text -SaveRequested(request UID, target UID, attribution, message, timing policy) +CommitRequested(request UID, target UID, attribution, message, timing policy) SaveRegistered(request UID, sequence, registration time, attach deadline) SaveAttached(request UID, window ID) WindowClosed(window ID, included inputs, reason) diff --git a/docs/design/commitrequest-save-wait-options.md b/docs/design/commitrequest-save-wait-options.md index 73851628f..bf33a53b1 100644 --- a/docs/design/commitrequest-save-wait-options.md +++ b/docs/design/commitrequest-save-wait-options.md @@ -505,8 +505,8 @@ the participant's message on the commit. - [`../spec/commitrequest-design.md`](../spec/commitrequest-design.md), the current contract. The code depends on it, so each phase changes that page in the same commit. -- [`../spec/commitrequest-admission-authorship.md`](../spec/commitrequest-admission-authorship.md), - where the submitter comes from and why the lookup is present-or-never. +- `commitrequest-admission-authorship.md`, since removed with the admission webhook: where the + submitter came from and why the lookup was present-or-never. - [`../spec/watch-event-ordering-and-attribution-grace.md`](../spec/watch-event-ordering-and-attribution-grace.md), the head-of-line wait this page proposes to wait behind rather than shorten. - [`attribution-removal-wait-options.md`](attribution-removal-wait-options.md), the same shape of diff --git a/docs/design/concurrent-author-windows.md b/docs/design/concurrent-author-windows.md new file mode 100644 index 000000000..0ebd89f7a --- /dev/null +++ b/docs/design/concurrent-author-windows.md @@ -0,0 +1,145 @@ +# Design: one commit window per author, open at the same time + +Status: **proposal, not built.** Raised by gitops-api's review of the source-side save (PR #456): +with saves created straight from a browser, a second person's edit splits the first person's change +into two commits. This page sets out the change that would fix that, what it costs, and what is +still open. + +## The problem + +A branch worker holds **one** open commit window at a time (`branchWorkerEventLoop.openWindow`). A +window belongs to one author on one `GitTarget`, and a write that is anyone else's cannot join it +(`openWindow.canAppend`). It closes the window first, with close reason `identity_change`, and opens +its own. + +```mermaid +sequenceDiagram + participant A as Alice + participant B as Bob + participant W as Branch worker + A->>W: edit cm/a + Note over W: window 1 opens: Alice + B->>W: edit cm/b + Note over W: window 1 closes (identity_change), window 2 opens: Bob + A->>W: edit cm/c + Note over W: window 2 closes, window 3 opens: Alice + A->>W: CommitRequest "fix(shop): cart limit" + Note over W: attaches to window 3 only +``` + +Alice's change is three files and one save. Git gets two commits for her: `cm/a` under the generated +message, `cm/c` under hers. Nothing fails; the history is just wrong about what belongs together. + +gitops-api avoids it today by serializing every save and its edits behind one lock in its server +(`commitGate`). A client that creates `RemoteSaveRequest`s from the browser has no such lock, so the +split comes back. + +Two saves by **one** author on one `GitTarget` are a different problem, and it is already handled. +A window carries one message, so the second save is refused with `SaveInProgress` while the first +still waits for a window or collects one. That stays true under this proposal, which changes only +how different authors interleave. + +## The proposal + +Keep a window per **(GitTarget, author, attribution class)** instead of one per worker. A write joins +the open window with its identity, or opens one. Writes by others no longer close it. Each window +keeps its own timers, its own attached request and its own message. + +```mermaid +sequenceDiagram + participant A as Alice + participant B as Bob + participant W as Branch worker + A->>W: edit cm/a + Note over W: Alice's window opens + B->>W: edit cm/b + Note over W: Bob's window opens; Alice's stays open + A->>W: edit cm/c + Note over W: joins Alice's window + A->>W: CommitRequest + Note over W: Alice's window: cm/a + cm/c, her message +``` + +Writes that name no author keep sharing one window, as today: they have no author to tell apart. + +## What has to stay true + +### One path, one order + +Today the single window gives a simple guarantee: commits land in the order their writes arrived, +so the last write to a path is the one Git ends with. With several open windows, that breaks unless +something keeps it: + +1. Alice edits `cm/x` (her window). +2. Bob edits `cm/x` after her (his window). +3. Bob's window closes first, then Alice's: Git ends with Alice's **older** `cm/x`. + +**Rule: a write to a path another open window holds closes that window first.** Bob's edit of `cm/x` +closes Alice's window, so her commit lands before his. The close reason is new, `path_conflict`. +The path is the window's own key (`windowPathKey`: the Git path, after placement), so a delete, a +rename to the same document, and a namespaced and a cluster-scoped object that map to one file are +all caught. Two authors editing the same object are therefore split exactly as today; two editing +different objects are not. + +### Everything that closes "the window" closes all of them + +The other close reasons are about the branch, not about one author, and each must close every open +window, oldest first: + +| Reason | Why all of them | +| --- | --- | +| `foreign_commit` | the branch moved under every window | +| `apply_due`, `resync_before_apply` | an apply must not run with unpublished edits from anyone | +| `shutdown` | nothing survives the loop | +| `buffer_limit` | the cap is the worker's memory, not one author's | + +`buffer_limit` could instead close only the largest or oldest window. Closing all is simpler and the +cap is rarely reached; this is an open question below. + +### Commit order is close order + +Commits land in the order their windows close, not the order their first writes arrived. With Alice +open since 10:00 and Bob since 10:01, Bob's commit can come first. Together with the path rule above, +no path ever goes back in time, but `git log` no longer reads as one timeline of edits. That is the +point of the change, and it needs saying in `docs/configuration.md` when it ships. + +### Saves + +- A save attaches to the open window of its identity, by lookup instead of comparison with the one + window. `matchesWindow` keeps its rules. +- `WindowMismatch` becomes rare: it meant "only another author's window was open". With every author + allowed a window, it remains only for the window a save may not claim: an unattributed one, without + `SavedBy` or `TakeOver`. +- `SaveInProgress` is unchanged: it is already per author and `GitTarget`. +- The takeover (`TakeOver`, `SavedBy`) claims the unattributed window, which stays one window. + +## What it costs + +- **The loop's state.** `openWindow` becomes a map with an opening order, the close timer becomes the + earliest of all windows' deadlines, and every path that reads `l.openWindow` today (attach, + finalize, the apply window, resync cut-off, metrics) has to say which window, or all. +- **Memory.** One window per active author. The byte cap stays global, so the bound does not change. +- **Tests.** The commit-window, attach and apply-window suites assume one window. Most of them keep + their meaning with one author; the interleaving cases need new ones, the path-conflict rule first. +- **Metrics.** `git_commit_windows_total` gains `close_reason="path_conflict"`. A gauge of open + windows would show how often authors overlap at all. + +## Alternatives + +1. **Keep one window, and let clients serialize.** What gitops-api does today. Correct, and costs a + server on the save path, which is what `RemoteSaveRequest` set out to remove. +2. **Per-author windows without the path rule.** Simpler, and wrong: a path can go back in time + with no error. +3. **Merge concurrent windows into one commit with several authors.** Git has one author per commit; + `Co-authored-by` trailers would blur exactly the attribution the product exists for. + +## Open questions + +1. Does `buffer_limit` close every window, or only the one that crossed it? +2. Should `identity_change` remain, for a write of another `GitTarget` on the same branch, or do + windows of different `GitTarget`s also stay open side by side? The proposal says side by side; + the per-`GitTarget` encryption and bootstrap that made one window per target necessary are per + window already. +3. Is a cap on open windows needed beyond the byte cap, such as one per `GitTarget`? +4. When a save's own window is closed early by `path_conflict`, it commits with the save's message + and less than the person meant to save. Is that acceptable, or should the save report it? diff --git a/docs/design/mutation-capture-lab-own-cluster.md b/docs/design/mutation-capture-lab-own-cluster.md new file mode 100644 index 000000000..bcbeb4f0e --- /dev/null +++ b/docs/design/mutation-capture-lab-own-cluster.md @@ -0,0 +1,239 @@ +# Mutation-capture lab on its own cluster + +Status: proposed, 2026-10-27. Not built. + +The [mutation-capture lab](../spec/mutation-capture-lab-design.md) records what Kubernetes emits +through watches, audit, admission, CRD conversion, and an aggregated API, and commits it as a +corpus under [`test/mutationlab/corpus/`](../../test/mutationlab/corpus/). That corpus is evidence +the product relies on: `internal/queue/fact_index_corpus_test.go` and +`fact_index_selector_corpus_test.go` replay it as fixtures. It is also how a new Kubernetes +version is judged: capture again, and the corpus diff is the behavioral changelog. + +This page designs the lab as a cluster of its own, which can run beside the e2e cluster, and lists +the requirements that keep the capture faithful and the lab working. + +## What broke, and why it could + +The lab never owned its wiring. `task lab-e2e` swapped the lab image into the product's controller +Deployment on the e2e cluster ([`swap-image.sh`](../../test/mutationlab/swap-image.sh)) and borrowed +everything around it: + +| The lab needs | It came from | Since `fd6f6933` | +|---|---|---| +| an admission webhook registration for `/validate-all` | the product's `ValidatingWebhookConfiguration` | deleted | +| a serving certificate, port 9443, and its mount | the product's `Certificate`, `Service`, and Deployment | deleted | +| the CRD conversion webhook (row 14) | the CA bundle and Service read off that registration | broken with it | +| audit deliveries | the API server's webhook, aimed at the product's NodePort and route, with the product's certificates | works | +| a pod, an identity, and RBAC | the product's Deployment and ServiceAccount | works, by accident | + +The driver's readiness check waits for an admission record, so the run now fails before its first +scenario, although audit still flows. + +The deletion only exposed the root cause: nothing runs the lab, so a product change could remove +what the lab depended on, and every check stayed green. + +## Decision + +The lab gets its own k3d cluster, `gitops-reverser-capture-lab`, which runs the lab and nothing of +the product. Every source the corpus records arrives the way it does on any cluster: + +- the API server's **audit webhook backend** posts batches to the lab over mutual TLS; +- a **`ValidatingWebhookConfiguration`** sends admission reviews to the lab's Service; +- the **CRD conversion webhook** of the scenario Widget CRD calls the lab's Service; +- the **sample aggregated API server** (`wardle`) serves flunders behind an `APIService`; +- the lab **watches** through its own ServiceAccount. + +Nothing is replayed, proxied, or read from a file. That is the point: a new Kubernetes version is +tested by creating the lab cluster on it, and only real deliveries say what that version does. + +A namespace on the e2e cluster cannot get there. The API server takes one audit webhook, set when +the cluster is created, and on the e2e cluster it belongs to the product. Sharing it brings back +the swap. + +## Design + +```text + host: task lab-capture ── go test driver ── port-forward :18081 ─────────────┐ + │ + k3d cluster gitops-reverser-capture-lab (one node, k3s pinned by the lab) │ + ┌────────────────────────────────────────────────────────────────────────────┼─┐ + │ kube-apiserver │ │ + │ audit webhook backend ── mTLS ── 127.0.0.1:30444 (NodePort) ─┐ │ │ + │ ValidatingWebhookConfiguration mutation-capture-lab ─────────┤ │ │ + │ CRD conversion (Widget) ─────────────────────────────────────┤ │ │ + │ APIService v1alpha1.wardle.example.com ── wardle-server │ │ │ + │ ▼ ▼ │ + │ namespace mutation-capture-lab: Deployment mutation-capture-lab │ + │ :9444 audit (mTLS) · :9443 /validate-all + /convert · :8080 records API │ + │ watches: configmaps, deployments, pods, widgets, flunders │ + └─────────────────────────────────────────────────────────────────────────────┘ +``` + +### The cluster + +[`start-cluster.sh`](../../test/e2e/cluster/start-cluster.sh) creates it, with: + +- `CLUSTER_NAME=gitops-reverser-capture-lab`, so the context is `k3d-gitops-reverser-capture-lab` + and its stamps live under `.stamps/cluster/k3d-gitops-reverser-capture-lab/`; +- `K3S_IMAGE` from `LAB_K3S_IMAGE`, which defaults to the image recorded in + [`CLUSTER.md`](../../test/mutationlab/corpus/CLUSTER.md), independent of the e2e cluster's; +- `AUDIT_DIR_REL` pointing at the lab's stamp directory, which holds the e2e audit policy, copied + from the one tracked file, and the lab's final audit webhook kubeconfig; +- `K3D_AGENT_COUNT=0`. The e2e cluster runs an agent because it restarts its server node to load + the audit kubeconfig; the lab never restarts, so one node is enough. + +The API server's audit flags, batch settings included, come from the script unchanged, so both +clusters deliver audit the same way. + +### Certificates, before the cluster exists + +The e2e cluster restarts its server because cert-manager issues the audit certificates only after +the cluster is up, and the API server reads its audit kubeconfig only at startup. The lab issues +its certificates first, with a small Go command, into the lab's stamp directory: + +- one CA; +- a serving certificate for `mutation-capture-lab.mutation-capture-lab.svc`, used by the audit, + admission, and conversion servers; +- a client certificate for the API server. + +The audit kubeconfig is then complete when the cluster is created: server +`https://127.0.0.1:30444/audit-webhook`, the CA, the client certificate, and the Service's DNS +name as `tls-server-name`. The lab requires and verifies that client certificate. The swapped lab +never did, because the swap passed no client CA. + +After creation, the serving certificate and CA go into the cluster as Secrets, and the CA is +written into the registration's `caBundle` when it is applied. The driver writes it into the Widget +CRD's conversion `caBundle` from the stamp directory. cert-manager is not installed: the lab's own +certificates have no rotation to manage, and cert-manager arrives on the e2e cluster only through +the whole Flux stack. + +### The deployment + +`test/mutationlab/deploy/`, a kustomization, holds: + +- namespace `mutation-capture-lab`; +- ServiceAccount `mutation-capture-lab`, and a ClusterRole that grants `get`, `list`, and `watch` + on what `--watch-resources` names, nothing else; +- the Deployment, one replica, so every audit batch reaches the one store; +- Service `mutation-capture-lab`: ClusterIP 443 to 9443 for admission and conversion, NodePort + 30444 to 9444 for audit, and 8080 for the records API; +- `ValidatingWebhookConfiguration` `mutation-capture-lab`, with the rules the corpus was captured + under: `resources: ["*"]` top-level only, `CREATE`, `UPDATE`, `DELETE`, `scope: "*"`, + `matchPolicy: Equivalent`, `failurePolicy: Ignore`, `timeoutSeconds: 1`. Rows 5 and 6 record no + admission for `/status` and `/scale` because those rules match no subresources; Kubernetes would + send them if asked. + +The sample aggregated API comes from +[`test/e2e/setup/manifests/sample-apiserver/`](../../test/e2e/setup/manifests/sample-apiserver/), +which gains a `kustomization.yaml` so the e2e setup and the lab apply the same directory. + +### Tasks + +The lab's tasks move into their own Taskfile, `test/mutationlab/Taskfile-lab.yml`, included flat +like the e2e one. The verbs follow the playground's: + +| Task | Does | +|---|---| +| `lab-bootstrap` | issues the certificates, creates the cluster, builds and imports the lab image, applies the deployment and the sample API server, and waits until every source is live | +| `lab-capture` | runs the scenarios against the lab cluster and compares the capture to the corpus | +| `lab-corpus-update` | runs the scenarios and writes the capture as the new corpus, `CLUSTER.md` included | +| `lab-status` | shows the cluster, its k3s version, the lab pod, and how many records it holds | +| `lab-cleanup` | deletes the cluster and its stamp directory | + +`lab-e2e`, `swap-image.sh`, and the lab's dependency on `_controller-deployed` are removed. + +Testing a new Kubernetes version: + +```bash +task lab-cleanup +task lab-bootstrap LAB_K3S_IMAGE=rancher/k3s:v1.38.0-k3s1 +task lab-corpus-update +git diff test/mutationlab/corpus/ +``` + +### Beside the e2e cluster + +Both clusters can run at once. What each one owns: + +| Concern | e2e | lab | +|---|---|---| +| cluster and context | `gitops-reverser-test-e2e` | `gitops-reverser-capture-lab` | +| stamps | `.stamps/cluster/k3d-gitops-reverser-test-e2e/` | `.stamps/cluster/k3d-gitops-reverser-capture-lab/` | +| API port on the host | picked by k3d | picked by k3d | +| NodePort 30444 | inside its own node container | inside its own node container | +| run lock | `e2e.lock` in its stamps | `lab.lock` in its stamps | +| host port-forward | Gitea, Prometheus, Valkey, Flux, Argo CD | records API on 18081 | +| k3s image | the e2e pin | `LAB_K3S_IMAGE` | + +`start-cluster.sh` switches the current kube context to the cluster it creates. The lab passes a +new `K3D_SWITCH_CONTEXT=false`, and the script's devcontainer kubeconfig rewrite reads the named +context rather than the current one. Otherwise bootstrapping the lab would move a person's +`kubectl`, or an e2e run started without `CTX`, onto the lab cluster. + +## What is reused + +| Piece | How | +|---|---| +| `start-cluster.sh` | as is, plus `K3D_SWITCH_CONTEXT` and a context-scoped kubeconfig rewrite | +| the audit policy | the same tracked file, so the corpus shows what the product's clusters audit | +| the audit flags and batch settings | from the script, unchanged | +| the sample aggregated API server | the same manifests, through a new `kustomization.yaml` | +| the lab binary, recorders, normalizer, corpus, and driver | unchanged, apart from the client-CA flag, the context, and the readiness check | +| the content-hashed lab image and its build | unchanged | +| `lab-port-forward.sh` | with the lab's namespace and selector | +| the audit kubeconfig template | from `generate-audit-webhook-kubeconfig.sh`, fed from files instead of Secrets | + +Not reused: `inject-webhook-tls.sh`, whose node restart the lab avoids, and the Flux stack. + +## Requirements + +Each one names the failure it prevents. + +1. **Real deliveries only.** Every record arrives from the API server or the aggregated API server + through its own mechanism: the audit webhook backend, a webhook registration, a CRD conversion + webhook, a watch. No log scraping, replay, or proxy. Otherwise a new Kubernetes version is + judged through a translation that may hide what changed. +2. **Mutual TLS on audit, verified.** The API server presents its client certificate and the lab + checks it, as a production receiver does. Otherwise the lab records deliveries such a receiver + would refuse. +3. **The registration's match rules stay those the corpus was captured under.** Otherwise rows 5 + and 6 change meaning while looking like a Kubernetes change. +4. **One audit policy file, shared with e2e.** Otherwise the corpus drifts from what the product's + clusters audit. +5. **Self-contained.** No lab manifest, script, or task names a product resource or depends on a + product task. A static check enforces it. Otherwise the next product change breaks the lab + unnoticed, as this one did. +6. **Coexistence.** The lab has its own cluster name, stamps, lock, and port, and never switches + the current kube context. Otherwise running it beside e2e corrupts one of the two. +7. **Every source live before the first scenario.** `lab-bootstrap` ends by proving audit, + admission, conversion, the aggregated API, and the watch each with a probe write it then finds + among the records. Today's check covers admission alone. +8. **The version is pinned and recorded by the tool.** `lab-corpus-update` writes the k3s image + and the server version into `CLUSTER.md`. Otherwise the corpus and its provenance disagree, as + they did when e2e moved to 1.37 and the corpus stayed on 1.36.4. +9. **Something runs it.** A scheduled CI job runs `lab-bootstrap` and `lab-capture`, and fails on + any corpus diff. Without it the other requirements hold only until the next unnoticed change. + +## Build order + +1. `K3D_SWITCH_CONTEXT` and the context-scoped rewrite in `start-cluster.sh`, and the + `kustomization.yaml` for the sample API server. +2. The certificate command, and the lab's client-CA verification. +3. `test/mutationlab/deploy/` and `Taskfile-lab.yml` with `lab-bootstrap`, `lab-status`, and + `lab-cleanup`. +4. The driver: the lab context, the CA from the stamps, and the five-source readiness check. Then + `lab-capture` and `lab-corpus-update`; delete `lab-e2e` and `swap-image.sh`. +5. A capture on the pinned k3s v1.36.4, committed on its own: its diff must be empty, or explained + line by line, because only the delivery path changed. +6. A second commit that moves the pin to the e2e cluster's version, so its diff is the Kubernetes + change alone. +7. The scheduled CI job, the static self-containment check, and the lab's README and + [spec](../spec/mutation-capture-lab-design.md#isolated-test-setup) rewritten to match. + +## Open questions + +- **CI cadence.** Nightly plus manual dispatch is the proposal. A per-PR job would catch a break + in the PR that causes it, for a few runner minutes per PR. +- **Memory beside e2e.** One k3s node with the lab and the sample API server should stay near + 1 GiB, but that is an estimate until measured. If it is more, `lab-capture` can take the e2e lock + as well, so the two run in turn. diff --git a/docs/design/support-boundary/admission-consent.md b/docs/design/support-boundary/admission-consent.md index 3416d1c2f..b6395875a 100644 --- a/docs/design/support-boundary/admission-consent.md +++ b/docs/design/support-boundary/admission-consent.md @@ -165,8 +165,7 @@ RBAC). This boundary is the same class of "no" as correctness — not lift­able Consent needs a synchronous surface, and admission is the only one: it is the sole point where the user's intent can be accepted or rejected **whole, before persistence** — the property the write-gating doc names as the real argument for the gate. The infrastructure exists (the operator -already runs admission webhooks, -[`validate_operator_types_handler.go`](../../../internal/webhook/validate_operator_types_handler.go)), +ran an admission webhook for `CommitRequest` authorship, since removed), and the preflight design specifies the baseline. Consent would add one branch: ```mermaid diff --git a/docs/finished/redis-key-schema-v3.md b/docs/finished/redis-key-schema-v3.md index ae2ad5765..ee04bed71 100644 --- a/docs/finished/redis-key-schema-v3.md +++ b/docs/finished/redis-key-schema-v3.md @@ -5,7 +5,7 @@ > Status: **IMPLEMENTED** — 2026-06-29. The `v2 → v3` key-format bump landed for the live Redis families: > the audit author index, the watch-resume cursor, and the new command-author **store** (storage > mechanics only — the CommitRequest admission/controller rewrite in -> [commitrequest-admission-authorship.md](../spec/commitrequest-admission-authorship.md) is a separate round and +> `commitrequest-admission-authorship.md`, since removed, is a separate round and > is **not** wired up yet). Scope: (1) replace the opaque `attr` namespace with a top-level `author` > domain; (2) re-lay audit-sourced resource author facts as a readable, **object-grouped** hierarchy > (`:object::` + `:last`, plus a type-scoped `:rv:` escape hatch), carrying @@ -24,7 +24,6 @@ > `internal/queue/attribution_index.go`, > [`internal/queue/redis_store.go`](../../internal/queue/redis_store.go), > [`internal/watch/author_resolver.go`](../../internal/watch/author_resolver.go), -> [CommitRequest authorship from admission](../spec/commitrequest-admission-authorship.md), > [attribution](../spec/attribution.md), > [watch event ordering & attribution grace](../spec/watch-event-ordering-and-attribution-grace.md), > [watch-first ingestion architecture](watch-first-ingestion-architecture.md). diff --git a/docs/future/gitops-api-open-asks.md b/docs/future/gitops-api-open-asks.md index 0d9927dec..2a03b48a8 100644 --- a/docs/future/gitops-api-open-asks.md +++ b/docs/future/gitops-api-open-asks.md @@ -1,94 +1,240 @@ -# The gitops-api team's open asks - -Status: **open asks from a production user**, condensed on 2026-10-24 from their standing list -(revision 15, 2026-10-20, written against `v0.53.0`). The asks keep their numbers. Closed asks are -left out: #30 shipped as `GitReceiver` (#432), #33 went with the prefetch in #425, #22 is fixed -(`pkg/manifestanalyzer/repo.go`), #26 and #27 shipped (`git_branch_targets`, -`commit_requests_total`), #25 shipped in 0.44.0 (#343), and #6 is answered by immutability that -fails at admission with "delete and recreate". Since revision 15: #32 is a row in -[the Flux guide](../bi-directional-flux.md#troubleshooting); #31 is -`apply_claimed_resource_reconciled_total`, labeled by applier kind and claim; ask #23 is pinned at the -fact index, for a finalizer cleared by a PATCH or an UPDATE, in either order -(`TestFactIndex_AFinalizerClearedByAControllerNamesTheDeleter`); and #11 shipped in 0.41.0 (#290): -create and update write through one encoder, `internal/yamlstyle`, and a test fails on a second -one. A file GitOps Reverser did not write is re-indented once, on its first edit. Kind and field -names are updated to `main` (`FluxClaim` for `FluxTrigger`, `KubeProvider` for `ClusterProvider`, -`ClaimedResourceReconciled` for `ResumedElsewhere`). - -Their suggested order, as of revision 15: #34 when the e2e budget allows; then #10; #28 when two -owners share a cluster. #28 is deferred: see below for why. - -## 34. An e2e for an in-cluster applier with a remote source cluster - -**Ask.** One e2e where the `GitTarget`'s `KubeProvider` names a remote cluster and the `FluxClaim`'s -names the operator's own, with an API edit landing during an apply. - -**Why.** It is their production topology: Flux runs in the management cluster and applies into a -remote workspace through `spec.kubeConfig`; GitOps Reverser reads that workspace remotely and holds -the `Kustomization` locally. They proved the hold, the apply cycle, the receiver and `ForeignCommit` -in it, but never saw the apply window withhold anything (`apply_window_collisions_total` stayed at -zero), so the part that reads `managedFields` on the remote object is untested in this topology. -An e2e that provokes one collision would settle it. - -Two smaller things they noted, both answered. `OperationUnresolved` on the `GitTarget` and the -`WatchRule` is not recorded again: each object's readiness takes the reason of the one it depends -on (`ApplierClaimReady` on the `GitTarget`, `GitTargetReady` on the rule), so one operation shows on -three objects by design. And `apply_requests_total{outcome="sent"}` counts only an applier that was -asked: an apply that is not needed is never sent, so it is not counted. - -## 28. Two operator instances cannot share one config-plane cluster - -**Ask.** A way for two instances sharing one config-plane cluster to partition the CR set, so that -exactly one instance reconciles a given `GitTarget`. Smallest useful first, and they stack: - -1. **`--watch-namespaces`**, a `Cache.DefaultNamespaces` pass-through. It partitions the namespaced - kinds, which since #430 includes `KubeProvider`; only `ClusterWatchRule` stays cluster-scoped. -2. **An explicit owner selector**, a label selector or a `spec.instance`-style field paired with an - `--instance-name` flag, which covers the cluster-scoped kinds too. - -**Why.** They hit it moving onto a cluster that already runs an instance for its owner's own -workloads: two populations, different source clusters, providers, credentials and namespaces, and -still both instances would reconcile both sets. RBAC is no workaround: controller-runtime issues a -cluster-wide List, so a narrowed ClusterRole fails the cache sync and the manager exits. - -**What sharing costs them**, for calibration: one Valkey holding both populations' attribution -facts; one upgrade cadence; one RBAC posture that is the union of both; and, customer-visible, a -restart for one owner's reasons costs the other's users their commit message, because -`RenderMatchesLive` starts `Unknown` after a restart and an open window is dropped. They are not -blocked: one shared instance works. - -**Deferred, not built.** Step 1 was built and dropped before merge (#445). It removed two of the -four costs above, the shared Valkey and the restart reaching the other owner's users. It could not -remove the other two: - -- **One upgrade cadence stays.** The CRDs are cluster-scoped, so both instances run against one - schema, and an API break (we make them without migrations) breaks whichever instance lags. -- **One RBAC posture stays.** The ClusterRole is still cluster-wide. - -It is also not the HA mechanism the chart's replica guard -(`charts/gitops-reverser/templates/validate-replica-count.yaml`) asks for. Two instances serving -disjoint namespaces never contend for a `GitTarget`; two replicas of one instance would, and need -leader or ownership coordination instead. Against that, it adds a lasting cost: a second tenancy -model beside one instance serving many namespaces through grants (`consumerNamespaces`, -cross-namespace provider references). Every cross-namespace reference gains a "namespace this -instance does not serve" failure, and every cluster-scoped kind needs its own ownership check. - -The cost their users see is the restart dropping an open window. That is better fixed inside one -instance, for every installation: the durable pending writes in [the TODO](../TODO.md). A second -owner who needs its own upgrade cadence or RBAC needs its own cluster; #28 would give it neither. - -## 10. Sibling inference can place an object in another namespace's directory - -**Ask.** Consider the namespace when inferring where a *new* document goes. Cosmetic. +# gitops-reverser in production: how we use it, and what we'd still like + +_From the [gitops-api](https://github.com/ConfigButler/gitops-api) team. **Revision 17, 2026-10-26** +(first sent 2026-07-09). Written to be readable from the gitops-reverser repo — no gitops-api +knowledge assumed. Every ask is traced to something we measured on a running system. **We run +`v0.54.0`.**_ + +## Where this stands + +**`0.54.0` closed every ask we had open but three, and we adopted it the day it was tagged.** It +reshaped every connection object with no in-place migration, so we rebuilt our cluster rather than +migrate it; loop conformance 21/21 and our editor click-through pass on it. #30 shipped as +`GitReceiver` and deleted more of our code than any release since #1: one receiver for the whole +platform, so our relay now forwards GitHub's delivery verbatim, with no per-repository token, Role +or receiver. Adoption write-up: [reverser-0.54-adoption.md](https://github.com/ConfigButler/gitops-api/blob/main/docs/design/reverser-0.54-adoption.md). + +**One new ask, #35, and unlike the last few it costs us something today.** `0.54.0` stopped +applying the forward leg after the reverser's own pushes, which is right for the cluster and leaves +the applier's own state — its applied revision, its source artifact — behind the branch until +somebody else pushes. We read that state to publish. Details and a proposal below. + +**We accept your reasoning for deferring #28**: a namespace split leaves the CRDs and the +ClusterRole shared, and the cost that reaches users (a restart dropping an open window) is better +fixed inside one instance — which is the second vote below. + +| Open | What it is | +|---|---| +| **35** 🟡 new | Converge the applier on the reverser's own pushes, at a bounded rate | +| **34** 🟡 | An e2e for an in-cluster applier with a remote source cluster — our production topology | +| **10** ⚪ | Sibling inference can place an object in another namespace's directory — cosmetic | +| **28** ⏸ | Two instances on one config-plane cluster — deferred, by agreement | + +Plus two **votes for items already on your TODO**, with our evidence: filtering `kube-root-ca.crt`, +and the durable worker queue. + +## Who we are + +**gitops-api lets an external user edit their config-as-data by talking to a Kubernetes API.** They +log in with GitHub, they `kubectl` (or click) against an API surface that is theirs, and their edit +becomes a **Git commit authored by them** in their own repository. gitops-api itself **never writes +Git** — that is entirely gitops-reverser's job, and it is why the product exists. The attribution +_is_ the product. + +Each tenant is a [kcp](https://kcp.io) **workspace**: a nodeless, near-namespace-cheap logical +cluster with its own CRDs and its own OIDC. + +```text + ┌───────────────────────────────────────────┐ + a human's edit ────▶│ kcp workspace (one per tenant) │ + (Dex/OIDC token) └───────────────┬───────────────────────────┘ + │ shard-wide audit firehose, tagged `kcp.io/cluster` + ▼ + ┌──────────────────────────────────────┐ + │ ONE gitops-reverser (STOCK) + ONE │ a KubeProvider per tenant; + │ Valkey, for the whole platform │ attribution partitioned by + └───────────────┬──────────────────────┘ spec.attribution.auditRoute + ┌─────────────┴─────────────┐ + ▼ ▼ + tenant A's repo tenant B's repo ← attributed commits + ▲ + │ Flux `Kustomization` with spec.kubeConfig → the workspace, + └─ held by a FluxClaim through the ONE in-cluster KubeProvider + (forward leg: git → cluster) +``` + +**We run the stock image, with no patches.** One reverser mirrors our management cluster in-cluster +and every tenant's workspace remotely. A GitHub App's single webhook reaches it through our server, +which forwards each push to one `GitReceiver`. + +We also **link** `pkg/manifestanalyzer`: our onboarding wizard's folder step runs your acceptance +gate so the folders it offers are the folders your writer will accept. The reverser is therefore +pinned twice for us — the image we run and the Go module we link — with a test that fails the build +when they part. + +--- + +## Open asks + +### 35. Converge the applier on the reverser's own pushes, at a bounded rate + +**Ask.** A periodic converge: when an applier's last applied commit is behind the branch tip, and +nothing else is pending for it, apply the tip — **at most once per interval**. For example +`FluxClaim.spec.convergeInterval` / `ArgoCDClaim.spec.convergeInterval`, defaulting to something like +`1m`, with `0` keeping `0.54.0`'s behaviour. The decision stays in the branch worker, like every +other one; the claim's own periodic reconcile may be the natural clock. Batched by construction: a +burst of saves inside one interval costs one apply, not one per push. + +It is a timer on a reason you already have. A `Converge` already applies the tip "because nothing +says the applier runs it", and already asks nothing when the applier's last succeeded operation +applied that commit. Today it fires once at worker start and once per new `GitTarget`; this asks for +it on a clock as well. + +**Why.** `applier-echoes.md` S2 is right about the cluster: after the flush, an apply following the +worker's own push changes no object, so it protects nothing there. But the applier's **own state** +is read by other things, and on `0.54.0` that state stops moving: + +- the `Kustomization`'s `status.lastAppliedRevision`; +- the `GitRepository`'s artifact. It fetches only when the claim asks it to apply, so it stays at + the last foreign commit as well; +- everything downstream of those two: Flux's commit-status notifications, `flux get kustomizations`, + any promotion or packaging step that consumes "what was applied". + +Measured on `0.54.0`, after one conformance run and one editor click-through: branch tip +`be3b08d2`, artifact **and** applied revision `c81500d0`, the last commit somebody else pushed — +**15 commits behind, and it moves only when somebody pushes in GitHub.** On `0.53.0` the quiet-branch +`CatchUp` kept them level. + +It costs us one feature. We publish each repository's folder as an OCI image from what the forward +leg **applied** (applied revision == artifact revision), downloaded from source-controller. On +`0.54.0` an edit made in our editor is mirrored, committed and attributed perfectly, and then never +published until an unrelated push happens. We cannot work around it from our side: the only lever +we have is to fetch or reconcile the Flux objects ourselves, and reconcile ownership tells us — rightly +— that a fetch of ours between your check and your resume applies a commit nobody checked. + +**Why not after every push.** That was `0.53.0`'s `CatchUp` minus the quiet period, and it would make +every save cost an apply. A bounded interval keeps the applier eventually level for whoever reads its +state, and the cluster pays at most one no-op apply per interval per applier. + +### 34. An e2e for an in-cluster applier with a remote source cluster + +**Ask.** One e2e where the `GitTarget`'s `KubeProvider` names a remote cluster (`secretRef`) and the +`FluxClaim`'s names the operator's own (`inCluster`), with an API edit landing during an apply. + +**Why.** It is our production topology: Flux runs in the management cluster and applies into a remote +workspace through `spec.kubeConfig`; the reverser reads that workspace remotely and holds the +`Kustomization` locally. We have proven the hold, the apply cycle, the receiver and `ForeignCommit` +in it (21/21 in our conformance probe, on `0.53.0` and on `0.54.0`). We have **not** seen the apply +window withhold anything in it — `apply_window_collisions_total` stayed at zero across every run — +so the part of #29 that reads `managedFields` on the remote object is untested in this topology by +anyone. An e2e that provokes one collision would settle it. + +### 10. Sibling inference can place an object in another namespace's directory — ⚪ cosmetic + +**Ask.** Consider the namespace when inferring where a _new_ document goes. **Why.** The cold-start layout is namespace-first, but once a layout exists, placement follows -siblings. Two tenants syncing a repository named `infra` produce two objects named `infra` in -different namespaces, and one lands in the other's directory. Nothing is corrupted, but -`git log -- tenants/tenant-a/` shows another tenant's history. - -## A possible #24: say when a stored field was pruned - -From their 0.43 upgrade: a deleted field is pruned by the API server, so the object stays -`Ready=True` and quietly uses a default or a wider scope. They had to write a test asserting the -*absence* of the old spellings. When an applied object still carries a field the CRD no longer -serves, a condition or `Event` on that object would make a migration self-diagnosing. Not filed as an ask. +siblings. Two tenants syncing a repo called `infra` produce two objects named `infra` in different +namespaces, and one lands in the other's directory. **Nothing is corrupted** — both documents and +both namespaces are correct, and deleting one removes exactly that document. Only the path suffers: +`git log -- tenants/tenant-a/` shows another tenant's history. It may well be intentional. + +### 28. Two reverser instances cannot share one config-plane cluster — ⏸ deferred, by agreement + +We asked for a way to partition the CR set between two instances on one cluster (`--watch-namespaces`, +then an owner selector), because we are moving onto a cluster that already runs a reverser for its +owner. You built the first step and dropped it before merge (#445): it removes the shared Valkey and +the cross-owner restart, but not the shared CRDs, the shared upgrade cadence or the shared +ClusterRole, and it adds a second tenancy model beside grants. **Agreed.** We run one shared +instance; the cost that would make us ask again is the restart below, and you want that fixed for +everyone. Our analysis: [reverser-instance-topology.md](https://github.com/ConfigButler/gitops-api/blob/main/docs/design/reverser-instance-topology.md). + +## Votes for your own TODO + +Both are already on your list. These are the measurements behind our vote, not new asks. + +**Filter `kube-root-ca.crt`** (`docs/TODO.md`, "Filter more cluster-generated noise"). Our tenants +mirror ConfigMaps, so every tenant repository carries the workspace's cluster CA certificate — a +file nobody wrote, that means nothing outside that one cluster. Every rebuild of the cluster gives +it a new CA, and the first snapshot commits the change as a `message_source="reconcile"` commit by +the committer. That is the only commit-without-a-human our alarm +(`git_commits_total{author_kind="committer"}`) sees, and it fires once per rebuild on every +tenant. A default filter, with an opt-back-in for whoever really wants it, would remove both. + +**The durable worker queue** (`docs/TODO.md`, "Make the worker queue durable"). This is the one cost +of sharing an instance that our users can see. `RenderMatchesLive` starts `Unknown` after a restart, +an open commit window is dropped, and a save made in that window loses the message its author typed +and its attribution. `0.54.0`'s #443 now holds live edits while fidelity is pending, which fixes the +edit; as far as we can tell the window's message is still dropped. It is the last item standing +between us and "a deploy of the reverser is invisible to the people editing". + +## Process notes + +- **Retire a field by rejecting it first.** `0.44.0` kept `eventTemplate`/`groupTemplate` in the + schema for one release with a CEL rule that refuses them, and `0.53.0` did the same for + `allowAnySourceNamespace`. Both failed loudly in our operator instead of silently changing + behaviour. When a field is simply removed, the API server prunes it and the object stays + `Ready=True`, so we write tests asserting the _absence_ of old spellings. Making + rejection-before-removal the default would make those tests unnecessary. +- **`0.54.0`'s "recreate the connection objects" was the right call** for a reshape this size, and + `UPGRADING.md`'s eight steps were complete. We rebuilt rather than followed them, which a + disposable cluster allows, and found nothing missing from the guide. + +--- + +## Shipped — the record + +One line each: what happened, and whether we are happy. + +| # | Ask | Shipped | Verdict | +|---|---|---|---| +| 1 | Separate the config plane from the watched cluster | `v0.38.0` (#249/#251), #259 | **Happy** — one reverser now serves our whole platform; it deleted a tier of our architecture | +| 3 | A Redis key prefix | #221 | **Happy** — defaulted to the historic root, no migration | +| 4 | Export `pkg/manifestanalyzer` | #221 | **Happy** — the foundation of our wizard's folder step | +| 5 | A `CommitRequest` that can assert an author | the admission webhook records the submitter (#220 itself closed unmerged) | **Happy** — a save's author comes from who filed it | +| 6 | Movable `GitTarget` destination, or fail loudly | immutability that fails at admission with "delete and recreate" | **Happy** — we delete and recreate, now for a `GitBranch` too | +| 7 | Degrade without `apiregistration.k8s.io` | #223, `v0.34.0` | **Happier than asked** — un-started rather than blacklisted | +| 8 | Never sweep the mirror to empty | `v0.39.0` (#260), `prune.mode: OnEvent` | **Very happy** — an API default, not a guard we maintain | +| 9 | RBAC that matches the runtime need | #223, `v0.34.0` | **Happy** | +| 11 | Serialize `[CREATE]` and `[UPDATE]` bodies identically | `0.41.0` (#290), one encoder | **Happy** — the mirror's `git diff` is readable | +| 12 | Strip `kcp.io/cluster` from Git writes | #227, `v0.35.0` | **Happy, and grateful** — fixed before we reported it | +| 13 | Tag releases so Go can resolve them | `v0.37.0` | **Happy** — what made #4 pinnable | +| 14 | Watch a differently-named namespace than the rule's own | `v0.39.0` (#259) | **Happier than asked** | +| 15 | Make a lost audit route loud | `0.44.0` (#341), `AuditFactsReceived` | **Very happy** — a latched condition tells _not yet_ from _never_ | +| 16–21 | The analyzer asks: permanence, the report contract, `Key()`, rendered types, `FleetRoot`, `ReadScope` | `0.40.0`/`0.40.1` (#273, #275) | **Very happy** — three better than asked, two corrections of our ask | +| 22 | `ReasonRefusedStructural` no longer claims "permanent" | `0.41.0` (#290) | **Happy** | +| 23 | Deletion-as-intent names the deleter, not the finalizer-clearer | `0.41.0` (#289), pinned for UPDATE and PATCH in `0.54.0` | **Happy** | +| 26, 27 | Join the pipeline's halves; count `CommitRequest` outcomes | `git_branch_targets`, `commit_requests_total` | **Happy** — better than asked | +| 29 | Wake the reconciler only for commits the reverser did not make | `v0.53.0`, reconcile ownership | **Happy** — the forward-leg race is gone for every writer ([write-up](https://github.com/ConfigButler/gitops-api/blob/main/docs/design/reverser-feedback-forward-leg-race.md)) | +| 30 | One receiver endpoint for a GitHub App | `0.54.0` (#432), `GitReceiver` | **Very happy** — deleted our per-repository tokens, Roles and HMAC relay | +| 31 | Count "resumed elsewhere" in a metric | `0.54.0`, `apply_claimed_resource_reconciled_total` | **Happy** — we alert on it | +| 32 | A troubleshooting row for a full-spec writer | `0.54.0`, the Flux guide | **Happy** | +| 33 | Say why `prefetch` is false | `0.54.0` (#425), prefetch removed | **Happy** | +| — | `WatchRule` readiness lag (never filed) | `0.44.0` (#343) | **Happy** — rule readiness is usable as an alert | + +**#2 — ignore writes by an identity or field manager. ⚫ WITHDRAWN 2026-07-14, our mistake.** The +forward leg's echo is self-terminating: the applier applies the commit the reverser just pushed, the +content matches, the echo commits nothing. We built the filter ourselves and **it caused a +two-and-a-half-minute flip-flop**, every commit unattributed. **Please never add +`excludeUsers`/`excludeFieldManagers` on our account** — +[why](https://github.com/ConfigButler/gitops-api/blob/main/docs/design/forward-leg-loop.md). The right +shape was the one you already had: sanitize the content, do not filter the writer. + +## What is _not_ upstream's problem + +So a wishlist does not become a blame list: + +- **The audit muxer** was our integration with kcp, not a reverser concern — and `v0.39.0` let us + delete it. +- **kcp's front-proxy** panicking on an anonymous request, and its per-workspace authenticator + answering `Unauthorized` for a minute after a fresh start. kcp's, not yours. +- **Broken label references in our own alerts**, found while mapping our dashboards onto a new + metric surface. Ours, every time — which is why we now test our rules against `0.54.0`'s labels. + +## Suggested order + +1. **#35** — the only open ask with a cost to us today, and a timer on a reason you already have. +2. **#34** whenever the e2e budget allows — our production topology, and the one part of #29 nobody + has seen work in it. +3. The two TODO items above, in whichever order your plan already has them. +4. **#10** — cosmetic, whenever. diff --git a/docs/future/source-side-save-plan.md b/docs/future/source-side-save-plan.md new file mode 100644 index 000000000..fb8ebbe32 --- /dev/null +++ b/docs/future/source-side-save-plan.md @@ -0,0 +1,322 @@ +# Plan: saving from the source cluster + +A `CommitRequest` saves the writes in its author's open commit window. It lives in the control +cluster, where the operator runs, while the edits it saves happen on the source cluster. This plan +adds a second way to save: a small request created **on the source cluster**, next to the edits, +attributed from that cluster's audit stream like the edits are. + +It also plans the second step: letting a save take over a window whose writes name no author, as an +opt-in, for a source cluster that cannot send audit events at all. There it is the only way a save can +put its author's name on the commit. + +Status: superseded. It was built on `feat/gitops-api-asks-converge-root-ca` as a second kind, +`RemoteSaveRequest` in `source.configbutler.ai/v1alpha1`, with a takeover for clusters that send no +audit events. The [review that replaced it](../review/20261027-save-request-simplification.md) folded +both kinds into one `CommitRequest`, served on the control cluster and on a source cluster through one +core, and removed the takeover with the admission webhook. The +[configuration reference](../configuration.md#serving-the-commitrequests-of-a-source-cluster) describes +what shipped; this page keeps the reasoning that led there. + +This revision takes in gitops-api's review of the first draft, and a later API review of the +implementation: see [After the API review](#after-the-api-review). + +## Decisions + +1. **Build the source-side save first.** It gives the right author wherever the source cluster has + audit, and an honestly unresolved one where it does not. +2. **It is a new kind, defined by GitOps Reverser.** A `CommitRequest` with another schema, or a + kind gitops-api defines and translates, were both considered. See + [Why a new kind](#why-a-new-kind). +3. **The takeover is the second step.** Some people cannot configure their API server's audit, so + for them it is the only way to get their name on a commit. It is opt-in on the GitTarget, which + chooses how much a save may claim; a request that asks without permission fails loudly. See + [The takeover](#the-takeover). +4. **`cf3b97be` does not merge as it is.** Its automatic takeover becomes the opt-in, or comes out. + +## Why: two identity domains + +Today the save and the edits are attributed in two places: the `CommitRequest`'s author comes from +the control cluster's admission webhook, the edits' authors from the source cluster's audit events. +They match only when one person has the same username in both. + +gitops-api makes them match. Its server impersonates the human when it creates the `CommitRequest`, +so the admission webhook records the same Dex email that kcp's audit records for the edits, and the +ordinary author rule attaches. gitops-api does not need a takeover. What it pays for that is +elsewhere: + +- **A grant into the control cluster.** Every tenant's identity may `create commitrequests` in its + `tenant-` namespace there. It is the one place a tenant reaches outside its kcp + workspace. +- **A server on the save path.** The browser cannot reach that namespace, so every save goes through + gitops-api's `/editor/commit`, which impersonates and creates the request. + +A save created in the workspace removes both. "A customer's only API surface is their kcp workspace" +then holds with no exceptions, and the browser creates the save itself. That is the production +benefit this plan is for. It also helps any setup where people can reach only the source cluster. + +```mermaid +flowchart LR + subgraph Source["Source cluster (kcp workspace)"] + Edit["Alice edits cm/a"] + Save["Alice creates a save"] + end + subgraph Control["Control cluster"] + Op["Operator"] + end + Edit -->|"watched, attributed
from audit"| Op + Save -->|"watched, attributed
from the same audit"| Op + Op -->|"status written back
(a new write on the source cluster)"| Save +``` + +## The source-side save + +### What it does + +The operator watches the new kind on each source cluster through its KubeProvider. It attributes +each request exactly as it attributes the edits, from the source cluster's audit facts. It then +attaches the request to the matching commit window, and writes the request's status back where its +author can read it. + +- **On a source with audit**, Alice's save and Alice's edits resolve to the same user, so the save + attaches under the ordinary author rule. No takeover and no trust are involved, and Git names the + person who made the edits. +- **On a source with no audit**, the save names nobody, like the edits. A request that names no actor + may attach to a window that names none, which is today's rule + (`TestPendingCommitRequest_UnresolvedRequestAttachesToUnresolvedWindow`). The save gets its message + and its timers, and the author in Git stays unresolved. The window still holds everyone's + unattributed writes, but nobody is named as their author. + +### Why a new kind + +gitops-api could define its own kind and translate it into a `CommitRequest`, but it cannot learn who +created the object: watching the workspace shows the object and nothing about its creator. Only the audit stream +says that, and GitOps Reverser is what reads it. A translation would also spend part of the attach +wait on an extra hop, and add a second status to keep in step. + +Reusing `CommitRequest` with a different schema would be worse than a separate kind: + +- **Another shape.** It names a GitTarget on another cluster than the one it is created on. +- **Another trust model.** Its author comes from audit; a `CommitRequest`'s comes from admission. +- **One name for two things** invites the confusion that `tenant-` already costs gitops-api. +- **Installed alone.** A workspace gets one small CRD instead of the whole `configbutler.ai` set, so + nobody is offered GitTargets that nothing reads. gitops-api installs CRDs into workspaces through its + `workspaceCRDs` list, so no `APIExport` is needed there. +- **Easy to never mirror.** In its own API group, or at least as its own kind, keeping it out of Git + whatever a `WatchRule` selects is one check, like `kube-root-ca.crt`. + +### Which GitTarget it saves + +It names one, in a required `spec.gitTargetRef` of `namespace` and `name` on the control cluster. +The people creating it need no access to GitTargets: their tooling supplies the address. This draft +had it name none and save every GitTarget mirroring its namespace; review showed that an unchanged +request then does more whenever another GitTarget is added, and that its caller cannot say "this one +only". See [After the API review](#after-the-api-review). + +### When its wait starts + +On a source with audit, a request is attributed only once its audit fact has joined its watch event, +and the edits it saves wait in the same audit pipeline (kcp batches audit with a `batchMaxWait` of +1s). The request's wait for a window starts when the branch worker registers it, which is after the +request itself was attributed. That is how a `CommitRequest`'s `attachTimeout` is anchored today +(`handleAttachCommitRequest` in `internal/git/commit_request_attach_loop.go`). The timeout then covers +only the spread between the request's stream and the edits' streams. Its +default can still be set for a source with audit, rather than copied from `CommitRequest`'s `2s`. + +### An unresolved save on a source that has audit + +On a provider whose `AuditFactsReceived` latch is `True`, a save that resolves no author means its +audit fact went missing. Attaching it to a window that names no author would claim exactly the +window that should stay unattributed. So the latch applies here too: + +```mermaid +flowchart TB + S["A source-side save is attributed"] --> N{"Did it resolve
an author?"} + N -->|yes| Own["Attach to that author's window"] + N -->|no| L{"Has its KubeProvider ever
received an audit fact?"} + L -->|no| None["Attach to a window that names no author"] + L -->|yes| Refuse["Refused: its fact went missing.
Ready=True with reason AuthorUnresolved"] +``` + +Refusing rather than waiting keeps the outcome visible, and the person saves again. + +### What it writes on the source cluster + +This is GitOps Reverser's first write on a source cluster. Today a source credential only lists and +watches the mirrored types ([rbac.md](../rbac.md)). The new access is scoped to the new kind: list, +watch, status update, and delete (for the finished-request TTL). It is off unless the KubeProvider +turns it on, and the KubeProvider's docs say what it adds. + +### Notes for clients + +- **Create it as JSON.** gitops-api found that kcp drops audit events whose request body is YAML. A + save sent as YAML would lose its author without any error. A browser sends JSON anyway; the docs + must say it for everyone else. +- **One save at a time per GitTarget and author.** gitops-api queues saves in its server + (`commitGate`). If the browser creates them directly, that queue is gone. A window carries one + message, so a second save of the same author on the same GitTarget, made while the first still + waits for a window or collects one, is refused at once with `SaveInProgress`, rather than losing + its message to the first in silence. The client saves again once the first resolved. A second + person's edit that lands while a window is open still closes that window early: windows are one at + a time per branch worker, whoever writes. That is + [a design question of its own](../design/concurrent-author-windows.md). + +## The takeover + +### Who needs it + +A source-side save names its author only when the source cluster sends audit events. Many people +cannot make it: a managed Kubernetes service usually does not let the customer point the API +server's audit at a webhook. On such a cluster no write and no save can name who made it, and Git +records every save as attribution-unresolved. A takeover is the only way to put a name on those +commits, and that name is a claim on trust by definition: the cluster cannot confirm it. + +### The risk + +On a cluster with no audit, every write names no author, so all of them share one window on the +GitTarget, whoever made them. A save that takes the window over names its submitter as the author of +everyone's writes. + +```mermaid +flowchart LR + subgraph Source["Source cluster with no audit stream"] + A["Alice edits cm/a"] + B["Bob edits cm/b"] + end + A --> Win["One window on the GitTarget
author: none"] + B --> Win + CR["Alice's save"] -.->|"a takeover would name Alice
as the author of Bob's edit"| Win +``` + +That is harmless when one person edits the GitTarget's namespaces at a time, and wrong as soon as two +do. Only the GitTarget's owner knows which it is, so the GitTarget decides. + +### What the GitTarget allows + +An enum, because there are two levels of trust worth offering: `unattributedWindows`, on each grant +in the GitTarget's `spec.allowedCommitRequests`. A request claims only what the grant admitting its +namespace allows; admitted by several, it may claim the most any of them allows. (The first version +was one field, `spec.commit.unattributedWindows`, for requests in the GitTarget's own namespace; +it moved into the grants when a `CommitRequest` could name a GitTarget in another namespace.) + +- **`"Keep"`** (the default): a save attaches only to its author's window, as today. On a cluster + with no audit, a save from the control cluster finds no window of its own. +- **`"SavedBy"`**: a save may attach its message to a window whose writes name no author, without + claiming them. The author stays unresolved, and the commit records who saved it: + + ```text + Saved-By: alice@example.com + ``` + + The message may still describe Bob's edit, but nobody is named as the author of something they + did not write. A misleading message is a much smaller harm than a false author. +- **`"TakeOver"`**: a save may take such a window over. Its submitter authors the commit, and the + message ends with a trailer that says the name was taken on trust: + + ```text + Author-Source: CommitRequest team-a/save-1; no audit record names who made these changes + ``` + +A grant on the KubeProvider was considered and set aside: the provider's owner usually does not know +who edits which namespace. The GitTarget's owner does, and grants it per consuming namespace. + +### Three conditions, all required + +```mermaid +flowchart TB + Start["A save finds an open window on its GitTarget
whose writes name no author"] --> Q1{"1. Did the save ask
for more than its own window?"} + Q1 -->|no| Own["Its own windows only"] + Q1 -->|yes| Q2{"2. Does the grant admitting it
allow what it asked (unattributedWindows)?"} + Q2 -->|no| Refuse["Refused: Ready=False, Stalled=True
reason TakeOverNotAllowed"] + Q2 -->|yes| Q3{"3. Has the KubeProvider ever
received an audit fact?"} + Q3 -->|yes| Own + Q3 -->|no| Take["SavedBy: attach the message, author unresolved
TakeOver: the submitter authors the commit"] +``` + +1. **The save asks for it.** An enum under `spec.window` beside `attach`, matching the GitTarget's: + `spec.window.author`, `"Own"` (the default), `"SavedBy"` or `"TakeOver"`. A save asks for what it + wants each time, and the grant bounds it: `"TakeOver"` allows a save that asks for `"SavedBy"`, + and not the other way round. +2. **The GitTarget's grant allows it.** A save that asks for more than the grants admitting it allow + fails at once, + when it is first reconciled: `Ready=False`, `Stalled=True`, reason `TakeOverNotAllowed`. The spec + is immutable, so it cannot recover by itself. +3. **The KubeProvider has never received an audit fact** (`AuditFactsReceived` is not `True`). On a + cluster that has sent audit, a write that names no author means its fact went missing, so no + save claims it, whatever 1 and 2 say. The save then attaches to its own windows only. + +### How it relates to the source-side save + +The two cover different clusters: + +| Source cluster | Source-side save | Control-cluster save with `"TakeOver"` | +| --- | --- | --- | +| Sends audit | The saver's name, from audit | Not needed; the latch blocks it | +| Cannot send audit | Message only, author unresolved | The saver's name, on trust | + +A source-side save cannot offer `"TakeOver"`: on a cluster with no audit, nothing records who created +it either, so it has no name to give. The name has to come from the control cluster's admission +webhook, which is why a takeover is always made by a control-cluster `CommitRequest`. + +## What happens to `cf3b97be` + +It takes over automatically. Most of it is the worker side of `"TakeOver"`: matching a window whose +writes name no author, authoring the commit as the submitter, and the `Author-Source` trailer. Two +ways forward before the branch merges: + +- **Turn it into the opt-in here.** Add `spec.commit.unattributedWindows` on the GitTarget and + `spec.window.author` on the `CommitRequest`, the controller's check and refusal, `"SavedBy"`, and + tests for each row above. The source-cluster e2e's save spec then sets both. +- **Take it out here**, and build the takeover in its own PR after the source-side save. + +## Open questions + +1. The source-side kind's name and API group. Proposed: `RemoteSaveRequest` in + `source.configbutler.ai/v1alpha1`. "Save" is the word its user means and the one the project + already uses for what a request asks; `SimpleCommitRequest` was considered and set aside, since + it is a different kind with another trust model, not a simpler `CommitRequest`. Its spec carries + no Git terms (a message, a wait, a catch-up), and its status still reports each GitTarget's + commit, so a save stays traceable. Its own group makes "never mirror it" and "install it alone" + one check each. +2. The default of its attach wait on a source with audit. +3. The reason name for an unresolved save on a source that has audit (`AuthorUnresolved` above). +4. Whether the operator should hold a second save on a GitTarget until the first has resolved. +5. Names for the takeover: `spec.commit.unattributedWindows` on the GitTarget and + `spec.window.author` on the `CommitRequest`, each `"Keep"`/`"Own"`, `"SavedBy"`, `"TakeOver"`. +6. `cf3b97be`: turn it into the opt-in on this branch, or take it out and build the takeover later? + +## After the API review + +A review of the implementation found three things: + +1. **A remote save is not confined to its namespace.** A window groups by GitTarget and author, not by + namespace, so a save from `shop` commits the same author's edits in `billing` too, or, with no + audit, anyone's. The review offered two answers: confine the save, or let the GitTarget's grant + authorize the whole window. The grant is the answer: an admitted `RemoteSaveRequest` saves its + creator's whole window on that GitTarget, and the docs say so. Confining it was built and taken out + again: a namespace-shaped exception to windows that are not namespace-shaped, with an outcome of its + own (`WindowOutsideNamespace`) that the person saving could do nothing about. +2. **A cluster-scoped watch matched every namespace.** A collection with no namespace is either + cluster-scoped or every namespace, and the match did not tell them apart. It now reads the + GitTarget's watched types and counts only namespaced ones. +3. **The GitTarget's owner could not consent.** Enabling the requests on a KubeProvider made every + GitTarget mirroring the namespace eligible, and a request steers a window's message, its timers + and an apply. The provider's switch still makes them reachable; `GitTarget.spec.allowedRemoteSaveRequests` + admits them, by source namespace (names or `"*"`; no `Same`, since the namespaces are another + cluster's) and an optional `requestSelector`. Omitted, it admits none: `NotAllowed`. Labels pick + requests and prove nothing about who made them. + +A second review then found that implicit target selection still made a save broader than its caller +may intend: a request reached every admitting GitTarget mirroring its namespace, so adding a +GitTarget changed what an unchanged request did. A `RemoteSaveRequest` now names its GitTarget in a +required `spec.gitTargetRef`, saves that one, and reports a flat status (`status.commit`, +`status.appliedRevision`, `Ready`, `Applied`). The request's namespace decides only admission. That +also removed a cold-start failure: with no named target, discovery that had not caught up reported +`NoGitTarget`. The same review fixed two recovery paths: a request served again after a restart +keeps the author found when it was first served (its status writes moved its resourceVersion, so its +audit fact no longer matches), and keeps an outcome already decided, following only its apply. + +A `CommitRequest` gained a cross-namespace `spec.gitTargetRef`, and with it +`GitTarget.spec.allowedCommitRequests`: a grant list like the others, defaulting to one `from: Same` +grant, whose grants also carry the `unattributedWindows` bound. The GitTarget that admitted a request +is recorded by UID (`status.gitTargetUID`), so a GitTarget created after the request under the same +name must admit it before anything is attached. diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index 7a03c6658..bcbbe616e 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -190,7 +190,7 @@ boundary, the commit, the push. Background: | Metric | Type | Labels | Notes | | --- | --- | --- | --- | -| `watch_events_total` | counter | `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource`, `outcome` | The ingest census: every delivered watch event, exactly once. `outcome` is `routed` / `unchanged` / `not_object` / `bookmark` / `shutdown` / `stream_error` / `route_failed`. | +| `watch_events_total` | counter | `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource`, `outcome` | The ingest census: every delivered watch event, exactly once. `outcome` is `routed` / `unchanged` / `cluster_generated` (`kube-root-ca.crt`, never mirrored) / `not_object` / `bookmark` / `shutdown` / `stream_error` / `route_failed`. | | `watch_event_handling_seconds` | histogram | `source_cluster`, `group`, `version`, `resource` | How long a stream was **busy** on one event, attribution wait included. Occupancy, not queue delay. | | `watch_sessions_ended_total` | counter | `source_cluster`, `group`, `version`, `resource`, `reason` | `closed` (the API server's routine watch timeout: the most common ending on a healthy cluster) / `expired` (cursor out of history, forcing a rebuild) / `error` / `stopped`. | | `watch_replay_duration_seconds` | histogram | `source_cluster`, `group`, `version`, `resource` | Time to `initial-events-end`: what a `410` storm charges. | @@ -216,7 +216,7 @@ boundary, the commit, the push. Background: | `git_next_retry_timestamp_seconds` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | When the next attempt at the pending writes is due, as a Unix timestamp. No sample while no retry is scheduled. | | `git_materialization_failures_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `reason` | Attempts to commit decided writes that stopped before any push: `reason` is `unreachable` or `parent_unavailable`. Nothing is lost; the writes wait for the retry. A failure `git_pushes_total` cannot see, because no push cycle started. | | `git_branch_targets` | gauge | `gitbranch_namespace`, `gitbranch_name`, `gittarget_namespace`, `gittarget_name`, `source_cluster` | Always 1. The **join** between the GitTarget-labeled half of the pipeline and the GitBranch-labeled half: the `GitTarget`'s `spec.gitBranchRef`. Published per configured GitTarget, whether or not its GitBranch exists or its worker runs. | -| `commit_requests_total` | counter | `outcome`, `gittarget_namespace`, `gittarget_name` | One per `CommitRequest` terminal **decision**. `outcome` is `committed` / `no_window` / `window_mismatch` / `already_present` / `failed`. A target that never resolved publishes the empty pair. See the counting note below. | +| `commit_requests_total` | counter | `outcome`, `gittarget_namespace`, `gittarget_name` | One per `CommitRequest` terminal **decision**. `outcome` is `committed` / `no_window` / `window_mismatch` / `already_present` / `save_in_progress` (refused: an earlier save of the same author on the target was in progress) / `author_unresolved` (no audit event named the request's creator, on a cluster that sends them) / `failed`. Requests served on a source cluster count here too. A target that never resolved publishes the empty pair. See the counting note below. | | `placements_total` | counter | `source`, `disposition`, `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource` | One per new document at a resolved path. | | `placement_refusals_total` | counter | `reason`, `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource` | One per new resource the writer declined. Every increment is a resource **absent** from the mirror. | | `placement_kustomization_entries_total` | counter | `outcome`, `gittarget_namespace`, `gittarget_name` | `added` / `no_change` / `failed`. | @@ -867,7 +867,8 @@ else pushed. See [who applies this folder](configuration.md#who-applies-this-fol | `apply_revision_mismatches_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `applier_kind`, `claim_namespace`, `claim_name` | One per apply operation that succeeded on another revision than the commit GitOps Reverser checked. A `Kustomization` applies whatever its `GitRepository` artifact holds when it reconciles; the artifact is read again just before the resume, so this is somebody fetching the source in between (a person, or the source's own interval). The apply window still withholds a late write over a newer edit; a steady rate means something else keeps fetching the source | | `apply_claimed_resource_reconciled_total` | counter | `applier_kind`, `claim_namespace`, `claim_name` | One per apply an applier claim did not start: a held `Kustomization` resumed (`flux resume`, or a controller writing its spec without `spec.suspend`), or a held `Application`'s automated sync switched on, by somebody else. From then until the claim takes it back, on its next look within a minute, the applier applies whatever its source holds, unchecked against the branch. The claim records a `ClaimedResourceReconciled` Event for each as well, but the API server folds identical Events into one and expires them, so alert on this counter. A `Kustomization` resumed so also shows the apply as an `Unresolved` operation on the claim | | `apply_window_collisions_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per write by the applier, while an apply it was asked for runs, to an object the API side edited after the flush before that request: the apply arriving late over a newer edit. `outcome` is `withheld` (proven to be the applier's write, by its field manager alone, with an entry as new as the edit or newer: `kustomize-controller`, or `argocd-controller` / `argocd-application-controller`; so not mirrored: Git keeps the API's value, and a `Repair` of the applier follows, which restores it on the cluster) or `ambiguous` (the applier and another field manager wrote in the same second, so it was mirrored as ever; if it was the apply, the revert is in Git as the applier's commit). An applier entry older than the edit is not counted: it is the API's undo of an edit, mirrored | -| `apply_echoes_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per write by the applier's own field managers alone, judged by the apply window: an apply GitOps Reverser asked for landing, object by object. `outcome` is `not_mirrored` (the applier wrote the object last, at or after the earliest send of the apply windows still open, and nobody edited it in the window: the apply itself, whose content is the commit Git already holds, so it is not mirrored) or `late` (the applier's managers alone wrote the object last with no apply window open: an echo arriving more than 10 seconds after its operation completed, or an apply somebody else started; it is mirrored as an edit). `not_mirrored` rises with every apply and needs no action. A steady `late` rate means applies keep landing outside the windows, and such a write can revert a newer Git-side change to the object: look for a person or a tool applying the folder by hand (`flux reconcile`, a manual sync), or for watch events reaching the worker long after the apply | +| `apply_echoes_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per write by the applier's own field managers alone, judged by the apply window: an apply GitOps Reverser asked for landing, object by object. `outcome` is `not_mirrored` (the applier wrote the object last, at or after the earliest send of the apply windows still open, and nobody edited it in the window: the apply itself, whose content is the commit Git already holds, so it is not mirrored) or `late` (the applier's managers alone wrote the object last with no apply window open: an echo arriving after its window closed, or an apply somebody else started; it is mirrored as an edit). `not_mirrored` rises with every apply and needs no action. A steady `late` rate means applies keep landing outside the windows, and such a write can revert a newer Git-side change to the object: look for a person or a tool applying the folder by hand (`flux reconcile`, a manual sync), or for watch events reaching the worker long after the apply | +| `apply_window_barriers_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per apply window whose operation completed: the window waits until the watch has delivered every write its GitTargets' source clusters held at that moment, so the applier's own writes are judged inside it however far behind the watch is (each event waits for its audit fact, and on a cluster that sends none, for the whole `--author-attribution-grace`). `outcome` is `met` (every stream delivered them; the window then closed at the end of its 10-second grace), `timed_out` (the window closed at the 2-minute limit with a stream still behind: a later write of the applier's is mirrored as an edit, and counted as `apply_echoes_total{outcome="late"}`), or `failed` (the source cluster could not be listed, or its resourceVersions are not numbers; the window closed after its grace alone). A steady `timed_out` or `failed` rate means the window cannot see every apply it asked for land | | `apply_ownership_conflicts_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | One per object another applier owns that the branch worker kept out of a coordinated folder: a live write not mirrored, or a snapshot object neither upserted nor swept. The owner is read from Flux's owner labels (`kustomize.toolkit.fluxcd.io/name` and `/namespace`) or Argo CD's tracking annotation (`argocd.argoproj.io/tracking-id`). Any non-zero rate is a misconfiguration: another `Kustomization` or `Application` applies objects into a namespace the folder mirrors, and two appliers applying one object fight each other. The worker logs each object once, naming both appliers; narrow the `WatchRule`, or move the object to one applier | | `apply_stale_snapshots_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per resync snapshot the branch worker found stale: collected while Git was ahead of the cluster (a commit pushed and not yet applied), collected across an apply, or arriving while Git is ahead. `outcome` is `discarded` (not decided; a fresh snapshot of its `GitTarget` is asked for once Git is no longer ahead, and the `GitTarget`'s live writes are held back by the render-fidelity gate until it lands) or `used` (the `GitTarget`'s snapshots had been held for 5 minutes while an apply kept failing, so this one was decided anyway, and whatever the unapplied commit changed in its objects is reverted, as the API-wins rule says). `discarded` comes in short bursts around applies and needs no action; a discard is not a resync failure and is not counted in `git_resync_failures_total`. Any `used` means an applier is failing for minutes at a time: read the applier claim's `status.operation` and the `ApplyFailed` Events. See [stale snapshots](spec/reconcile-via-watchlist-mark-and-sweep.md#stale-snapshots-are-discarded) | | `apply_operations_unresolved` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | Apply operations on the branch that have not completed 5 minutes after they were requested. Each is still followed, holds its applier's apply window open, and holds back the applier's next request until it completes, so a value that stays above zero is an applier stuck mid-apply: read its status, and the `ApplyUnresolved` Event on the `GitTarget`s. Read at scrape time | diff --git a/docs/quickstart.md b/docs/quickstart.md index 5e2041269..f24b65882 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -10,26 +10,14 @@ cluster, which the starter target names as its source. ![Config basics: a WatchRule or ClusterWatchRule selects what a GitTarget mirrors; the GitTarget writes its folder through a GitBranch and a GitProvider with a credentials Secret, and reads the cluster through a KubeProvider. The direction is always from the cluster into Git](images/config-basics.excalidraw.svg) -**Prerequisites:** a Kubernetes cluster with `kubectl`, Helm 3, and cert-manager for TLS. +**Prerequisites:** a Kubernetes cluster with `kubectl` and Helm 3. The default install renders no +certificate, so it needs no cert-manager; [named users](#want-named-users-on-your-commits) do. To write the objects yourself instead of letting the chart generate them, start from [`config/samples/`](../config/samples/): one manifest per object, consistent enough to apply together. [`configuration.md`](configuration.md) explains every field. -## 1. Install cert-manager - -Skip this step if cert-manager is already installed and healthy. - -The controller mounts an admission certificate at startup, so cert-manager must be healthy *before* -you install the chart. (`--set servers.admission.enabled=false` drops the dependency; the -[chart README](../charts/gitops-reverser/README.md) covers bring-your-own certificates.) - -```bash -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.3/cert-manager.yaml -kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s -``` - -## 2. Create a Git repo and a deploy key +## 1. Create a Git repo and a deploy key Create an (empty) repository the operator will write to, then generate a key. Add the public half as a **deploy key with write access**; the private half becomes the Secret in the next step: @@ -39,7 +27,7 @@ ssh-keygen -t ed25519 -C "gitops-reverser@cluster" -f /tmp/gitops-reverser-key - # Add /tmp/gitops-reverser-key.pub to your Git provider as a deploy key (write access) ``` -## 3. Create the demo namespace and Git credentials +## 2. Create the demo namespace and Git credentials Do this **before** installing. The starter `GitProvider` is only re-checked about every 5 minutes, so if the Secret is missing at install time your first commit can be minutes late; having it ready up @@ -72,7 +60,7 @@ credentials Secrets are accepted as-is (they must have **write** access). See [`github-setup-guide.md`](github-setup-guide.md) for the full GitHub guide and HTTPS/PAT fallback. -## 4. Install GitOps Reverser with the demo enabled +## 3. Install GitOps Reverser with the demo enabled Point the starter `GitProvider` at your repo and install: @@ -89,14 +77,14 @@ Replace `OWNER/REPO` with your repository (angle brackets would be parsed by the This install has these defaults: -- **No Redis**, so warm-restart cursors and author capture stay inactive. Add one with +- **No Redis**, so warm-restart cursors and author attribution stay inactive. Add one with `--set queue.redis.addr=HOST:PORT` (plus `queue.redis.auth.existingSecret` if it needs auth). - **Cluster-wide read on every watchable type, including Secrets** (`rbac.watchTypes.mode=any`). [`rbac.md`](rbac.md) explains how to narrow it. - **SOPS is on for the starter target**, so a `sops-age-key` Secret is generated in the demo namespace with a backup reminder. Only matters once you mirror Secrets; back it up if you keep the demo. -Wait for the controller. On a fresh install this waits on cert-manager issuing the certificate: +Wait for the controller: ```bash kubectl rollout status deployment/gitops-reverser -n gitops-reverser --timeout=300s @@ -134,7 +122,7 @@ A `Reason` that names a missing or unauthorized dependency (`KubeProviderNotFoun streams still settling is initialization, and resolves on its own. `describe` prints every condition with its message, which is where the specific cause is written. -## 5. Test it +## 4. Test it ```bash kubectl create configmap test-config --from-literal=key=value -n gitops-reverser-quickstart-demo @@ -166,3 +154,14 @@ To tear the demo down: `helm uninstall gitops-reverser -n gitops-reverser` and Enable audit attribution to use the Kubernetes actor as the Git author while keeping the committer unchanged. The [attribution setup guide](attribution-setup-guide.md) covers receiver configuration, API server delivery, and verification. It requires control over kube-apiserver settings. + +The audit ingress serves TLS with certificates cert-manager mints, so install cert-manager first, +and wait until it is healthy *before* you enable attribution: the controller mounts the audit +certificates at startup. Skip this if cert-manager is already installed and healthy, or bring the +Secrets yourself with `servers.audit.tls.certManager: false`; the +[chart README](../charts/gitops-reverser/README.md) covers bring-your-own certificates. + +```bash +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.3/cert-manager.yaml +kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s +``` diff --git a/docs/rbac.md b/docs/rbac.md index 76e370927..bee52cff4 100644 --- a/docs/rbac.md +++ b/docs/rbac.md @@ -86,6 +86,23 @@ evaluated against `Namespace`s in the *source* cluster, so a remote provider's i grant it exactly the namespaces and types the rules name — a `sourceNamespace: "*"` item asks for a cluster-wide `list`/`watch` of its types, and gets a clean 403 if the credential cannot serve it. +### `commitrequests` on a source cluster + +A source credential only reads, with one opt-in exception. A `KubeProvider` whose +`spec.commitRequests` is `Reconcile` serves the +[`CommitRequest`s](configuration.md#serving-the-commitrequests-of-a-source-cluster) created on its +cluster, in the namespaces it lends its `GitTarget`s, and writes their status back there. Its +credential then needs, in the `configbutler.ai` group and in each of those namespaces, `get`, `list` +and `watch` on `commitrequests`, and `update` on `commitrequests/status`. +`ReconcileAndDeleteFinished` also deletes finished requests, so it needs `patch` and `delete` on +`commitrequests` too. Nothing else on a source cluster is written, the CRD included. The manager +role carries the same rules for the cluster the operator runs in. + +Whoever may create a `CommitRequest` in a namespace may save their own commit window on any +`GitTarget` whose `spec.allowedCommitRequests` admits it: by `sourceNamespaces` on a source cluster, +by `consumerNamespaces` on the cluster GitOps Reverser runs in. That grant, not the provider's +switch, is the `GitTarget` owner's consent. + ## The operator does not read Secrets wholesale It holds **no Secret informer**: Secrets are excluded from the manager's cache diff --git a/docs/review/20261026-apply-window-late-events.md b/docs/review/20261026-apply-window-late-events.md new file mode 100644 index 000000000..aaed75daa --- /dev/null +++ b/docs/review/20261026-apply-window-late-events.md @@ -0,0 +1,269 @@ +# The apply window misses late events from a cluster with no audit stream + +On a source cluster that sends no audit facts, every event the watch delivers waits the full +attribution grace (10s in e2e) before it reaches the branch worker. The stream resolves events one at +a time, so the waits add up. The apply window closes on a wall clock, 10s after the applier reports +the apply finished, so the applier's own writes can reach the worker after the window has closed. +A write that arrives then is mirrored as an edit. If that write was an apply reverting a newer API +edit, the revert lands in Git and the edit is lost. + +This was found by the source-cluster e2e for gitops-api's production topology (ask #34): Flux in the +operator's cluster applies into a kcp workspace, and the GitTarget mirrors that workspace through a +`secretRef` KubeProvider. The e2e kcp has no audit stream. The run is at commit `cf3b97be`. + +Status: fix 2 below is implemented (`internal/git/apply_barrier.go`, `internal/watch/apply_barrier.go`), +with one change from the first draft: the mark is the newest resourceVersion among the collection's +objects, not the list's own, so a quiet stream does not wait for a bookmark. + +## The apply window, in short + +When the branch worker asks an applier to apply a commit, it opens an **apply window** for that +applier. Every live event for a GitTarget the applier covers is judged against it: + +- A write by the applier's field manager alone, to an object nobody edited since the window opened, + is **the apply landing**. Its content is the commit's, which Git already holds, so it is not + mirrored (`apply_echoes_total{outcome="not_mirrored"}`). +- A write by the applier's field manager alone, to an object the API side edited after the window + opened, is **the apply arriving late over that edit**. It is withheld, and a Repair puts the edit + back on the cluster (`apply_window_collisions_total{outcome="withheld"}`). +- Any other write is an edit, and is mirrored as usual. + +The window opens when the attempt is sent (`internal/git/applier_claim.go`, `openApplyWindow`) and +closes `applyWindowGrace` (10s) after the operation completed (`settleApplyWindow`). With no window +open, a write by the applier's manager alone is mirrored as an edit, and counted as +`apply_echoes_total{outcome="late"}`. + +```mermaid +flowchart LR + Send["Worker sends the attempt"] -->|window opens| Open(["Apply window open"]) + Open --> Done["Applier reports the operation completed"] + Done -->|"+ 10s grace"| Closed(["Apply window closed"]) + + Event["A live event arrives at the worker"] --> When{"Is a window open
when it arrives?"} + When -->|yes| Judge["Judged: the apply landing,
withheld, or an edit"] + When -->|no| Mirror["Mirrored as an edit
(late, if Flux wrote it)"] +``` + +The window is judged by **when the event arrives at the worker**, not by when the write happened on +the cluster. That gap is the whole problem. + +## Why events from the workspace arrive late + +Between the watch and the worker, each event waits for its author: an audit fact that names who made +the write. On a cluster with audit, the fact usually arrives within a second or two. On a cluster +with no audit stream, no fact ever arrives, so every event waits the full grace +(`--author-attribution-grace`, 10s in e2e) and then goes on unattributed. In this run, the median of +`attribution_resolution_wait_seconds` was 10s. + +A watch stream handles its events in order, one at a time (`internal/watch/target_watch.go`, +`attachAuthor` is called inline). So an event queued behind others waits for their waits as well. + +```mermaid +sequenceDiagram + participant K as kcp workspace + participant S as Watch stream + participant A as Attribution + participant W as Branch worker + + K->>S: write 1 (Flux) + K->>S: write 2 (an API edit), 1s later + S->>A: who wrote 1? + Note over A: no audit fact ever comes
waits the full 10s + A-->>S: unattributed + S->>W: event 1, 10s after write 1 + S->>A: who wrote 2? + Note over A: waits another 10s + A-->>S: unattributed + S->>W: event 2, 19s after write 2 +``` + +## What this run observed + +The collision spec pushed a Git-side change (`o1` to `during-other`), waited until the FluxClaim +showed the apply had been asked for, and then edited `during` through the workspace's API. These +are the worker's log lines, with the times the writes happened inferred from the spec's log. + +```mermaid +sequenceDiagram + participant T as Test + participant W as Branch worker + participant F as Flux + participant K as kcp workspace + + T->>W: push o1 (commit a9cb18d) + W->>F: 16:36:16 apply a9cb18d (ForeignCommit) + Note over W: window opens + F->>K: about 16:36:16, writes during-other + T->>K: about 16:36:17, edits during + F-->>W: about 16:36:20, operation completed + Note over W: 16:36:30 window closes
(completion + 10s) + K-->>W: 16:36:32, Flux's write to during-other arrives + Note over W: no window open:
mirrored as an edit, late echo counted + K-->>W: 16:36:42, the edit of during arrives
(25s after it was made) + Note over W: mirrored and committed +``` + +Two things happened here: + +- **Flux was faster than the edit.** It wrote before the spec's edit landed, so it never reverted the + edit. There was nothing to withhold, the edit reached Git and stayed on the workspace, and + `apply_window_collisions_total` stayed at 0. That is why the spec failed: it asserts that one + collision is counted. Waiting for the operation token and then editing does not guarantee that + the edit lands before the applier writes. The local E5 spec uses the same technique and passes + whichever side wins, which is why it never asserts the counter. +- **One of Flux's writes arrived after the window closed.** `apply_echoes_total{outcome="late"}` is 1 + on this branch. Here it was harmless, because the write held what Git already held. + +## What goes wrong when Flux does revert an edit + +Had the edit landed before Flux's write, Flux would have applied the commit it was asked for, which +does not hold the edit yet, and so reverted it on the workspace. The window exists to catch that +write. With the delays above, the write reaches the worker after the window has closed: + +```mermaid +sequenceDiagram + participant U as Someone editing + participant W as Branch worker + participant F as Flux + participant K as kcp workspace + participant G as Git + + W->>F: t+0 apply commit C (does not hold the edit) + Note over W: window opens + U->>K: t+1 edit: value = "edit" + F->>K: t+2 apply C: value = "old" (reverts the edit) + F-->>W: t+4 operation completed + K-->>W: t+11 the edit arrives + Note over W: window open: recorded as an edit,
mirrored + W->>G: commit value = "edit" + Note over W: t+14 window closes + K-->>W: t+21 Flux's revert arrives + Note over W: no window open: mirrored as an edit + W->>G: commit value = "old" + Note over G,K: Git and the workspace both hold "old".
No Repair is asked for. The edit survives
only in Git history. +``` + +This is the loss the apply window was built to prevent (#29), and from such a cluster nothing in the +window can catch it. + +A longer grace does not fix it. Events wait in a queue, so the delay grows with the number of events +ahead: the edit in this run waited 25s, more than twice the grace, because it queued behind Flux's +write. No fixed grace covers every burst. + +## Fix 1: do not wait for audit facts that never come + +The KubeProvider's `AuditFactsReceived` condition says whether an audit fact has ever arrived on its +route. When it is not `True`, waiting for one is pointless, so attribution could skip the wait for +events from that provider. Events would then reach the worker within moments of the write, inside +the window. + +```mermaid +flowchart LR + E["Event from the watch"] --> Q{"Has this KubeProvider ever
received an audit fact?"} + Q -->|yes| Wait["Wait up to the grace
for this write's fact"] + Q -->|no| Skip["Go on unattributed
at once"] + Wait --> W["Branch worker"] + Skip --> W +``` + +It is small, and it also removes the 10s or more of delay on every edit mirrored from such a cluster. +It has two limits: + +- On a newly added audited provider, writes made before its first fact arrives are recorded with no + author. With the CommitRequest takeover (`cf3b97be`), a request could then take over their window. +- It does nothing for a cluster that does send audit facts but sends them slowly. The window still + closes on the clock. + +## Fix 2: close the window by how far the watch has got + +The idea in one sentence: instead of closing the window "10 seconds after Flux said it was done", close +it "once the watch has delivered everything that happened on the cluster up to the moment Flux said it +was done". + +Every write to a Kubernetes object gets a **resourceVersion**, a position in the cluster's log of +changes, and a watch delivers events in that order. So the worker can mark a position and wait until +the watch has passed it: + +1. **Take a mark when the apply completes.** The worker lists the watched collection and takes the + newest resourceVersion among its objects, say `1284`: the position of the latest write to anything + in it. Every write Flux made for this apply happened before the apply completed, so each has a + resourceVersion of at most `1284`. +2. **Keep the window open until the watch passes the mark.** The stream records how far it has got + each time it records its cursor: after every event it handled, routed or not, and every + **bookmark**, a progress-only event in which the API server says "you have everything up to + resourceVersion X". Once it is at or past `1284`, it puts an answer on the worker's queue. +3. **Then close.** The answer rides the worker's queue behind every event the stream routed before + it, so once every stream has answered, every write Flux made has reached the worker, however long + attribution held each one. The window then closes at the end of its grace, or at once if that has + passed. + +The list's own resourceVersion would be the cluster's current position, which a quiet stream passes +only on a bookmark, and a bookmark can be a minute away. An object's resourceVersion is a write the +stream has seen or will see. + +```mermaid +sequenceDiagram + participant F as Flux + participant K as kcp workspace + participant S as Watch stream + participant W as Branch worker + + W->>F: apply commit C + Note over W: window opens + F->>K: writes (resourceVersions 1281, 1284) + F-->>W: operation completed + W->>K: list the collection + K-->>W: newest object resourceVersion 1284, the mark + Note over W: window stays open until the
stream passes 1284 + S->>W: event at 1281 (after its attribution wait) + Note over W: judged inside the window + S->>W: event at 1284 (after its attribution wait) + Note over W: judged inside the window + S->>W: the stream's answer: at 1284 + Note over W: past the mark: everything Flux
wrote has arrived. Window closes. +``` + +The decision the worker makes, for each window: + +```mermaid +flowchart TB + Done["Operation completed:
mark = the newest object resourceVersion now"] --> Check{"Has every stream of the
applier's GitTargets handed over
an event or bookmark at or past the mark?"} + Check -->|yes| Close(["Close the window"]) + Check -->|"not yet"| Bound{"Held longer than
the upper bound?"} + Bound -->|no| Check + Bound -->|yes| Give(["Close anyway, and log and count it"]) +``` + +What it costs: + +- **resourceVersions are opaque in the API contract.** Clients are not promised that they compare as + numbers, though the API server's own watch cache compares them that way on etcd. A source whose + resourceVersions do not parse as a number fails the barrier, and its window closes after its grace + alone, as before. +- **A write Flux deleted is not in the list**, so its event can still arrive after the window. A + DELETE is never withheld anyway, and the object it removes is one Git no longer holds. +- **A list per stream per apply.** It is paged, and made once when the operation completes. +- **A GitTarget can have several streams**, one per watched collection, and the applier can cover + several GitTargets. The window waits for all of them. +- **An upper bound** keeps a stream that is disconnected or stuck from holding the window open + forever. It is two minutes. Past it the window closes, and the close is logged and counted + (`apply_window_barriers_total{outcome="timed_out"}`), so the miss is visible. +- **A window held open longer** judges more writes. A write by Flux alone to an object nobody edited + is treated as the apply landing and not mirrored. That is still correct for an apply nobody asked + for, because its content is what Git holds. + +Fix 2 is the general fix: it covers a cluster with no audit stream, a cluster with slow audit, and a +burst of events of any size, because it waits for the watch itself rather than guessing a delay. +Fix 1 can come on top of it, to cut the delay on every mirrored edit from a cluster with no audit +stream. + +## The spec, separately + +Neither fix makes E5's way of provoking a collision deterministic: it edits once the apply was asked +for, and Flux often writes within a second of the resume, before the edit lands. The source-cluster +spec uses the delay this document is about instead. It edits first and pushes a Git-side change +right after. The edit's event waits out the attribution grace, so the push is heard, its apply asked +for and applied before the edit reaches the worker. The applied commit does not hold the edit, so +Flux reverts it, and with fix 2 the window still judges that write when it arrives, and withholds +it. That holds while the attribution grace (10s in e2e) outlasts the time from the push to the +apply request. diff --git a/docs/review/20261027-save-request-simplification.md b/docs/review/20261027-save-request-simplification.md new file mode 100644 index 000000000..2be9ad39c --- /dev/null +++ b/docs/review/20261027-save-request-simplification.md @@ -0,0 +1,935 @@ +# Review: one CommitRequest, no admission webhook + +Review of `feat/gitops-api-asks-converge-root-ca` (PR #456, at `70d07d91`), on one question: how +much of the `SaveRequest` / `RemoteSaveRequest` machinery can go, and in particular whether the +admission webhook can go entirely. + +Status: design decided on 2026-10-27; phase 1 is built in PR #456, phase 2 of the CRD signal is not. +The first version of this review proposed one kind on the source cluster only, named `SaveRequest`. +The decisions below replace that proposal. Line references are to `70d07d91`, before the change. + +## Decisions + +1. **One kind, named `CommitRequest`, in `configbutler.ai/v1alpha3`.** That is its name on `main`, + so the rename to `SaveRequest` (`b86dffbe`) is undone, which removes one breaking change. +2. **It is served in two ways, through one shared core.** + - **On the control cluster** it is an operator kind like any other. A controller in the + operator serves it with the operator's own identity, and a GitTarget admits it by + `consumerNamespaces`, default `from: Same`. + - **On any other source cluster** it is reached through the GitTarget's KubeProvider. That + KubeProvider sets `commitRequests` to `Reconcile` or `ReconcileAndDeleteFinished`, and the + GitTarget admits the request by `sourceNamespaces`. +3. **`spec.gitTargetRef` follows the API's reference convention.** An omitted namespace means the + request's own namespace, as in every other cross-namespace reference. On a source cluster, the + namespace is required, for the reason a ClusterWatchRule requires it: there is no + control-cluster namespace to default to. See [Naming the GitTarget](#naming-the-gittarget). +4. **The admission webhook goes completely.** That covers both handlers, the `/validate-all` e2e + endpoint, the command-author Redis store, the chart values, the manifests and the docs. + Takeover (`SavedBy`/`TakeOver`) goes with it, because only admission could name a taker. On + both paths, a request's author comes from audit facts, the same way an edit's author does. +5. **cert-manager stays**, but only as a prerequisite of attribution. See + [cert-manager](#cert-manager). +6. **GitOps Reverser never writes the CommitRequest CRD on a source cluster.** It reports the CRD's + state on the KubeProvider. Creating the CRD when it is absent is + [future work](#future-work-creating-the-crd), and updating it is ruled out for good. +7. **Several identities on one cluster are a goal**, so the path for other clusters works with + credentials scoped to namespaces. See + [Several identities on one cluster](#several-identities-on-one-cluster). + +## The design + +### The CommitRequest + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: CommitRequest +metadata: + name: raise-replicas + namespace: shop # on the control cluster, or on the GitTarget's source cluster +spec: + gitTargetRef: + namespace: team-shop # a control-cluster namespace; omitted, this request's own + # namespace, which only the control-cluster path accepts + name: shop + message: "Raise the replica count" + window: # defaults shown + attachTimeout: 5s + maxDuration: 2s + catchUp: AsDefinedOnClaim # or AfterPush +status: + author: jane@example.com # from audit; empty when no audit event names the creator + branch: main + commit: 3f2a… + conditions: [Ready, Stalled, Applied] +``` + +- **The spec and status** are today's `RemoteSaveRequest` ones, under the name and group of + `main`. `gitTargetRef` is a `meta.NamespacedObjectReference`, like every other cross-namespace + reference. +- **The phase reasons and the `save_requests_total` metric** come from today's control-cluster + reconciler. +- **Gone from the `main` version:** `spec.window.author`, because takeover is removed. +- **Kept from the `main` version:** `spec.window.attach` (`CurrentOrNext`/`Next`) and + `spec.whenNothingToCommit` (`Resolve`/`CommitEmpty`). Today only the control-cluster reconciler + has them. They move into the shared core, so a request on a source cluster gets them too. +- **Retention keeps `main`'s contract on both paths.** When a request finishes, it is stamped with + `configbutler.ai/delete-after`, and it is deleted once that time has passed. Removing the + annotation keeps the request. The source-cluster path's own TTL, which is computed from the + conditions, goes away. On a source cluster, deleting is opted into with + `commitRequests: ReconcileAndDeleteFinished` (see [The KubeProvider](#the-kubeprovider-speccommitrequests)). + +#### Naming the GitTarget + +Every reference to another namespace follows one rule: the namespace may be omitted, and then it is +the referring object's own namespace. The one exception is a ClusterWatchRule, which must name the +namespace because it has none of its own. + +| Reference | Type | An omitted namespace means | +| --- | --- | --- | +| GitTarget `gitBranchRef`, `kubeProviderRef` | `NamespacedObjectReference` | the GitTarget's namespace | +| GitBranch `gitProviderRef` | `NamespacedObjectReference` | the GitBranch's namespace | +| FluxClaim `kubeProviderRef`, `kustomizationRef`; ArgoCDClaim `kubeProviderRef`, `applicationRef` | `NamespacedObjectReference` | the claim's namespace | +| ClusterWatchRule `gitTargetRef` | `NamespacedObjectReference` | nothing: required, "a ClusterWatchRule has no namespace to default to" | +| WatchRule `gitTargetRef` | `LocalObjectReference` | always the WatchRule's namespace | +| `CommitRequest` `gitTargetRef` on `main` | `LocalObjectReference` | always the request's namespace | + +So `CommitRequest` follows the rule: + +- **On the control cluster, the namespace is optional**, and defaults to the request's namespace. + This is the behavior of `main` and of every other reference, and a single-cluster user writes + what they write today. +- **On a source cluster, the namespace is required**, for the ClusterWatchRule's reason. + - The request's namespace belongs to another cluster, so it is not a control-cluster namespace + to default to. + - Defaulting to "the control-cluster namespace with the same name" would turn a naming + coincidence into a reference. [definitions.md](../definitions.md) exists to prevent exactly + that. + - The KubeProvider's `{consumer.namespace}` does bridge the two sides by name, but only as a + variable someone writes on purpose, never as a silent default. + +**Where the rule is enforced.** Both clusters use the same CRD, and a CEL rule cannot know which +cluster it runs on, so the schema allows an omitted namespace. + +The source-cluster path does not pick up a request without a namespace, and it writes nothing to +it. It cannot refuse such a request, because it cannot know the refusal is true. A KubeProvider +whose kubeconfig points back at the control cluster sees the control cluster's requests, and there +an omitted namespace is valid. + +This is not a far-fetched case. `inCluster` is honored only for providers the installer lists in +`--in-cluster-kube-providers`, so a tenant who wants its own identity on the operator's cluster +needs exactly such a kubeconfig. That is the "several identities on one cluster" goal. A source +cluster user who forgets the namespace therefore sees no status. The field's doc says so, and +gitops-api always sets the namespace. + +**Rejected: a separate CRD variant for source clusters with the namespace required in CEL.** The +check would move into the schema. But one kind would then have two schemas, and the chart's +variant applied to a workspace would silently undo the check. + +### Which path serves a request + +The named GitTarget's KubeProvider alone decides which path serves a request: that path is the +request's **owner**. Two rules keep several observers of one request safe: + +1. **A path writes only what it knows to be true.** Only the owner serves a request and writes + its progress. An observer that is not the owner writes only `GitTargetNotFound`, after + `attachTimeout`, and only into an empty status. It writes nothing else, because nothing else is + true for every observer. +2. **Every status write is conditional on the resourceVersion its decision was made from.** On a + conflict, the writer reads the request again and decides again. Today's writer reads the object + again and overwrites it without deciding again + (`internal/watch/remote_save_requests.go:689-701`). An observer that sees a status it did not + write stops. So a `GitTargetNotFound` that was true when it was decided can land, but it never + overwrites an owner that has started. +3. **The owner claims a request by persisting its first status, and attaches only after that.** + The first write records the author and `Ready=Unknown`, reason `Saving`, and it is conditional + on the resourceVersion the owner read. Only when it succeeds does the owner attach to the + worker. If it conflicts and the fresh object already holds an outcome, such as a non-owner's + `GitTargetNotFound`, the owner stops without saving. + - Today's `serve` logs a failed status write and continues into `finalize` (l.348-352 and + l.391-393). Fixing the writer alone would therefore not be enough. + - **After the claim, a conflict re-applies the owner's outcome. It never restarts the save.** + Once the claim has landed, no other observer writes, by rule 1. So a later conflict comes only + from the owner's own writes, and the worker's outcome, once produced, is written onto the fresh + object as it is. + - **Ownership is structural, not "this process wrote the status".** After a restart, the owner + is still the path the GitTarget's KubeProvider picks. It finds its own `Saving` status and + resumes, as the server does today. + +| The request is on | The GitTarget's KubeProvider | What happens | +| --- | --- | --- | +| the control cluster | `inCluster` | **the control-cluster path serves it** | +| the control cluster | a kubeconfig, with either `Reconcile` value | left alone, because that kubeconfig may point back at the control cluster, and then its watcher serves the request | +| the control cluster | a kubeconfig, with `commitRequests: Ignore` | `NotAllowed`: "GitTarget … mirrors another cluster; create the CommitRequest there, and set `commitRequests: Reconcile` on its KubeProvider" | +| a source cluster that provider P watches | P, with a kubeconfig | **P's watcher serves it** | +| a source cluster that provider P watches | any other provider | left alone | +| a source cluster | `gitTargetRef` has no namespace | not picked up, and no status ([Naming the GitTarget](#naming-the-gittarget)) | +| either | the GitTarget does not exist | `GitTargetNotFound` after `attachTimeout`, by rule 1 and rule 2 | + +- **A control-cluster request for a GitTarget that mirrors another cluster is refused**, in every + install mode. Its author would come from the control cluster's audit, and the edits' authors + from the other cluster's. Those are two identity domains (F3), and the request would only work + while attribution is off. +- **"Local" means the provider's `inCluster` arm.** A kubeconfig that points back at the control + cluster counts as another cluster, because the operator cannot tell the difference. +- **Known gap.** A control-cluster request whose GitTarget's provider has `Reconcile` and points + somewhere else gets no status from anyone. The doc for the field says so. A cluster ID would + close the gap ([Future work](#future-work)). + +### What both paths share + +Once a path owns a request, the same core runs. These are its responsibilities, in order: + +1. **Admission** by `allowedCommitRequests`. A request no grant admits fails with `NotAllowed`. +2. **The author.** It comes from audit facts on the GitTarget's KubeProvider's route, from the + same index and with the same grace as the edits. + - **The core acquires the fact stream** for that route and `commitrequests`, as + `RemoteSaveRequestServer.run` does today (l.200). Today's control-cluster reconciler never + acquires one, because admission gave it the author. Without the stream, the fact follower + never reads the request's audit facts, so every save on that path would wait out the grace and + end without an author. + - **The wait runs whenever attribution is on**, and is skipped only when it is off (no + `AuthorResolver`). Whether a provider has received audit facts is checked after the wait, as + `sentAuditFacts` does today, and decides only `AuthorUnresolved`. Skipping the wait on a + provider that has not received a fact yet would lose the author while the provider's first + audit event is still in transit, or while its status condition lags. + - **The author is written to status in the first status write.** The lookup matches the + request's resourceVersion, and every status write changes it, so a later lookup could never + match. Today's server already does this (`status.Author` before the `Saving` write). +3. **Attach and outcomes:** one save in progress per author per GitTarget (`SaveInProgress`), the + UID binding, recovery after a restart, the phase reasons, the metric, and the worker's + `git.AttachSaveRequest`. +4. **Following the apply:** `catchUp: AfterPush` and the `Applied` condition. +5. **Retention:** stamping `configbutler.ai/delete-after`, and deleting once it has passed. + +Every status write follows rule 2 of [Which path serves a request](#which-path-serves-a-request). + +Each path feeds this core: + +- **The control-cluster path** is a controller-runtime reconciler. It uses the operator's cache, + RBAC and status client, and is cut down from today's `saverequest_controller.go`. +- **The source-cluster path** is today's `RemoteSaveRequestServer`. It runs a dynamic watch per + KubeProvider and writes status through that provider's credential. + +The core is written once, so F4 cannot come back. If the two paths ever need different logic +beyond picking up a request and writing its status, treat that as a design smell. + +**The default watch policy excludes `configbutler.ai/commitrequests`**, in place of +`source.configbutler.ai/remotesaverequests`. A request can sit in a namespace a WatchRule mirrors, +and it must never land in Git. + +What a save attaches to, in each install mode: + +| Install | Edit author | Request author | The request saves | +| --- | --- | --- | --- | +| Default (no attribution) | committer | nobody, no wait | the GitTarget's one unnamed window | +| Attribution on, a source cluster whose first audit event has not arrived | from audit, once it arrives | from audit, within the same grace | its creator's own window | +| Attribution on, single cluster | from audit | from audit | its creator's own window | +| kcp with audit (the managed product) | from audit | from audit | its creator's own window | +| A source cluster that sends no audit | nobody | nobody | the unnamed window: the message is kept, and the commit has no named author | + +Who may save is decided by `allowedCommitRequests`. Who authored the commit is decided by audit, or +by nobody. Neither is decided by admission. + +### The GitTarget: `spec.allowedCommitRequests` + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +spec: + allowedCommitRequests: + # The default when omitted: CommitRequests in this GitTarget's own namespace. + - consumerNamespaces: {from: Same} + # Requests on the control cluster, from other namespaces, by label: + - consumerNamespaces: + from: Selector + selector: {matchLabels: {team: shop}} + # Requests on the source cluster that spec.kubeProviderRef names: + - sourceNamespaces: ["*"] # every namespace this GitTarget mirrors +``` + +- **One list, and each grant sets exactly one of two fields.** A CEL rule enforces this. The two + fields are the ones the API already uses for each side: + - **`consumerNamespaces`** has the same type and meaning as in every other `allowed*` list: + control-cluster namespaces, with `from: Same | Names | Selector | All`. It admits requests on + the control-cluster path. + - **`sourceNamespaces`** has the same meaning as the KubeProvider's + `allowedGitTargets[].sourceNamespaces` and a WatchRule's `rules[].sourceNamespace`: namespaces + of the GitTarget's source cluster, as exact names or `"*"`. It admits requests on the + source-cluster path. + - Here `"*"` means every namespace this GitTarget mirrors. + - Variables are refused, because `{consumer.namespace}` would be the request's own namespace, + which bounds nothing. + - It can admit only namespaces the KubeProvider lends this GitTarget in `allowedGitTargets`, + the same bound its WatchRules have. +- **A grant on the side the GitTarget does not use admits nothing.** An `inCluster` GitTarget + never serves through `sourceNamespaces`, and any other GitTarget never serves through + `consumerNamespaces`. A `NotAllowed` message says which field applies. +- **Omitted, the list is `[{consumerNamespaces: {from: Same}}]`, and `[]` admits nobody.** This is + the pattern of the other lists: an API-server default, no `omitempty`, a Go mirror, and + `TestGrantDefaultsMatchCRDs`. A single-cluster user writes nothing, as on `main`. A user of + another cluster writes a `sourceNamespaces` grant, and that list replaces the default. +- **`requestSelector` is dropped.** It picked requests by labels their creator chooses, so it + proved nothing. It can come back later as an additive field. +- **`remote` is not used in any field name.** In this API, "remote" means the Git remote + (`status.remote`), and [definitions.md](../definitions.md) fixes "control cluster" and "source + cluster" as the two sides. + +### The KubeProvider: `spec.commitRequests` + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: KubeProvider +spec: + secretRef: {name: shop-workspace} # a kubeconfig Secret + commitRequests: ReconcileAndDeleteFinished # Ignore (default) | Reconcile | ReconcileAndDeleteFinished +``` + +Each value makes this cluster a bigger source of CommitRequests, and needs more of the credential +on `commitrequests` in the `configbutler.ai` group: + +| Value | What GitOps Reverser does | The credential needs | +| --- | --- | --- | +| `Ignore` (default) | nothing | nothing | +| `Reconcile` | serves the requests in the namespaces this KubeProvider lends its GitTargets, and writes their status. A finished request stays, and no `delete-after` is stamped on it, because nothing would act on it | `get`, `list`, `watch`; `update` on `commitrequests/status` | +| `ReconcileAndDeleteFinished` | as `Reconcile`, and also stamps `configbutler.ai/delete-after` when a request finishes, and deletes the request once that time has passed | as `Reconcile`, plus `patch` and `delete` | + +- **Why deletion is a separate value.** Deleting an object on someone else's cluster is a bigger + write than updating its status. The KubeProvider's owner lends the cluster, so the provider is + where that consent belongs, and the credential's RBAC follows it. gitops-api uses + `ReconcileAndDeleteFinished` for kcp workspaces. +- **Not `ReconcileAndPrune`.** [definitions.md](../definitions.md) reserves "prune" for + `GitTarget.spec.prune.mode`, which removes documents from a Git folder. +- **How long a finished request is kept stays installation-wide.** That is + `--save-request-ttl`, renamed back to `--commit-request-ttl`, its name on `main`: + default `48h`, and `0` stamps nothing in any mode. The annotation on a single request still + overrides it: remove it to keep the request, or edit it to move the deletion. +- **The annotation counts only while deletion is on.** If a provider goes from + `ReconcileAndDeleteFinished` back to `Reconcile`, annotations that were already stamped stay, but + nothing deletes those requests any more. +- **A delete the credential may not make** leaves the request in place. The + `CommitRequestsServed` message says so, and the condition stays `True`, because serving works. +- **The control cluster has no switch.** A CEL rule refuses every value but `Ignore` together with + `inCluster`. CommitRequests on the control cluster are always served, and finished ones are + deleted by the same flag and annotation, as on `main`. The operator owns that whole lifecycle, as + it does for every other kind. +- **I chose an enum over a boolean `reconcileCommitRequests: true`.** The API conventions avoid + booleans, and this third value is the reason: it was added without a second field. +- **It replaces `remoteSaveRequests: Enabled|Disabled`.** Today `Enabled` serves the requests and + also deletes them, as [configuration.md](../configuration.md#serving-the-remotesaverequests-of-a-cluster) + now lays out in a table of the same shape. The PR that builds this design replaces that table + with the one above. + +Both `Reconcile` values add one condition and one status field. A second condition comes in phase +2: + +- **`CommitRequestsServed`**: whether requests are being served, including whether the CRD is + there. +- **`status.commitRequests.namespaces`**: what this KubeProvider watches. +- **Phase 2: `CommitRequestCRDUpToDate`**, whether the CRD's schema matches the operator's. + +None of them exists on an `inCluster` provider. On the control cluster, the chart owns the CRD, as +it owns every other CRD, and the operator watches with its own permissions. + +#### Phase 1: the `CommitRequestsServed` condition + +This ships in the main PR. It needs only discovery and the watch itself. + +| Status | Reason | When | +| --- | --- | --- | +| True | `Watching` | the message says how many namespaces, or "every namespace" | +| True | `NothingToWatch` | no GitTarget mirroring through this KubeProvider is lent a namespace | +| False | `CRDNotInstalled` | discovery shows no `commitrequests.configbutler.ai` | +| False | `VersionNotServed` | the CRD is there, but does not serve `v1alpha3` | +| False | `Forbidden` | `list` or `watch` on CommitRequests was denied; the message names the namespaces | +| Unknown | `Unreachable` | discovery could not be read | + +- **Each message ends with its fix.** + - `CRDNotInstalled` and `VersionNotServed`: `kubectl apply --server-side -f` with the release's + `commitrequest-crd.yaml`. That is a new release asset. The existing `crds.yaml` holds every + CRD, and a source cluster needs only this one. + - `Forbidden`: the missing verbs and namespaces. + - `Unreachable`: the discovery error. + - `Watching` and `NothingToWatch` need no fix. +- **When it runs:** on every KubeProvider resync, and whenever the spec changes. A CRD applied by + hand therefore clears the condition without a restart. +- **Not part of `Ready`.** Mirroring works without it, as it does without `AuditFactsReceived`. +- **What it replaces.** Today a missing CRD shows up only as a log line, "The watch of + RemoteSaveRequests ended; reconnecting". +- **A printer column, `Saves`,** shows the reason, at `priority=1`. + +#### Phase 2: knowing whether the CRD is out of date + +This is its own PR. It is the "outdated" signal you asked for. It is deferred because it adds a +compatibility protocol, and because one part of it depends on kcp in a way that is not yet verified. + +**The revision line itself ships in phase 1**, in the type's doc comment, with its hash test. It is +a comment and a test, so it costs almost nothing. Without it, every CRD installed from phase 1's +release asset would carry no revision, and phase 2 could not judge any of them. + +The CRD carries a schema revision, and the operator compares it with its own. + +- **Where the revision is.** The last line of the kind's description reads + `Schema revision 7, compatible from revision 5.` kubebuilder renders the type's doc comment into + the CRD, so `kubectl explain commitrequest` prints the line. It is also in the cluster's published + OpenAPI v3 document. +- **How the operator reads it.** It reads `/openapi/v3/apis/configbutler.ai/v1alpha3`. The default + `system:discovery` role lets every authenticated identity read that document, so the check needs + no permission on CRDs and works with a credential scoped to a namespace. There is no fallback to + reading the CRD itself: an API that a kcp `APIBinding` provides has no CRD in the consuming + workspace ([kcp docs](https://docs.kcp.io/kcp/v0.28/concepts/apis/exporting-apis/)). +- **The gate.** The PR starts by verifying that kcp serves per-workspace OpenAPI v3, including for + APIs provided through an `APIBinding`. If it does not, the design changes before any code is + written. +- **The revision** goes up with every change to the generated CRD. A unit test hashes the CRD and + fails when the hash changes but the revision does not, the same way `hack/crdfields` guards the + field list. +- **The floor ("compatible from")** is raised by hand when a change breaks an older reader or + writer: a new required field, a removed field, or a changed meaning. This project makes hard + cuts, so the floor will often move with the revision, which is fine. +- **The comparison.** Take the operator's revision O and floor Fo, and the CRD's revision C: + - C equal to O: current. + - C below O: served when C is at least Fo. + - C above O: **never served**. +- **Why a newer CRD is never served.** The status writer replaces the whole `status` map, so an + older operator would erase every status field that a newer schema added. Serving a newer CRD + safely would need rules for preserving unknown fields, and tests for them. Refusing it costs one + operator upgrade, and that is the fix anyway. The CRD's own floor is therefore never read. +- **Rejected: comparing the served schema field by field.** It needs no bookkeeping, but it cannot + see changed validation, defaults or meaning, and a condition message cannot explain its result + well. + +| Status | Reason | When | Requests served? | +| --- | --- | --- | --- | +| True | `Current` | the CRD's revision is the operator's | yes | +| False | `Outdated` | older than the operator, but at or above the operator's floor | yes | +| False | `Incompatible` | older than the operator's floor | no | +| False | `NewerThanOperator` | newer than the operator; upgrade the operator | no | +| Unknown | `RevisionUnknown` | there is no revision line, so compatibility cannot be judged. This is not taken as evidence that the CRD came from elsewhere | yes | +| Unknown | `Unreadable` | the OpenAPI document could not be read | unchanged | + +- **`Outdated` keeps serving.** A user should not lose saves because a revision moved without + breaking anything. Only a crossed floor stops serving. +- **A reason that stops serving** also sets `CommitRequestsServed=False`, with reason + `CRDNotUsable`, pointing here. +- **The operator never updates the CRD**, whatever this condition says, for the reasons in + [Future work](#future-work-creating-the-crd). The condition is the signal, and a person acts on + it. + +#### When `commitRequests` is `Ignore` + +- **The conditions and `status.commitRequests` are removed.** + - A condition that would always say "not applicable" is noise. + - A `True` left over from before the switch would be wrong. + - `AuditFactsReceived` already works this way: it is absent while attribution is off. +- **The watches stop. The CRD and the requests on the cluster stay as they are.** A request created + there stays without status, unless another KubeProvider serves it. +- **Rejected: keep reporting the CRD read-only while the switch is off.** The check is cheap, but + it reports on something the KubeProvider's owner said it has nothing to do with. With several + providers on one cluster, every one of them would report the same state. + +### Several identities on one cluster + +On the control-cluster path, there is one identity: the operator's. This section is about the +source-cluster path. A KubeProvider is a cluster and an identity in one object, and several +KubeProviders can point at one cluster with different credentials, for example two kubeconfigs for +two teams' ServiceAccounts. + +| Concern | Today | With this design | +| --- | --- | --- | +| Which provider serves a request | every enabled provider tries, and the first one decides (F11) | the GitTarget's own provider only ([Which path serves a request](#which-path-serves-a-request)) | +| Watch scope | one cluster-wide list and watch, so a credential scoped to a namespace gets `Forbidden` | the namespaces the provider lends its GitTargets; a `Role` in each is enough | +| The CRD | cluster-scoped, so a credential scoped to a namespace cannot `get` it | read through OpenAPI, so every provider reports its state | +| Audit facts | providers in one namespace can share a route. A provider in another namespace that uses the same route is `Stalled`, reason `AuditRouteConflict` | unchanged | +| Duplicate work | each provider runs its own watch | each provider watches only its own slice; slices that overlap are watched twice | + +#### Chosen: the provider's lent namespaces set the scope + +A KubeProvider with `Reconcile` watches CommitRequests only in the namespaces it lends to the +GitTargets that mirror through it. These are the `sourceNamespaces` of the `spec.allowedGitTargets` +grants that admit each of those GitTargets, with `{consumer.namespace}` resolved per GitTarget. + +- **If any grant lends `"*"`**, the provider runs one watch across the cluster. Otherwise it runs + one watch per namespace. +- **`status.commitRequests.namespaces`** shows the result, so the derived scope is never hidden. +- **The credential already reads these namespaces for mirroring.** Its `Role` in each one gains + `get`, `list` and `watch` on `commitrequests`, and `update` on `commitrequests/status`. With + `ReconcileAndDeleteFinished` it also gains `patch` and `delete` on `commitrequests`. +- **A request in a namespace that is lent but not admitted still gets `NotAllowed`** from the + GitTarget's provider, so a wrong grant is visible. +- **Overlapping slices are harmless**, because only the GitTarget's provider acts. + +It needs no new field: the provider's owner chooses which namespaces to lend in +`allowedGitTargets`, and the GitTarget's owner chooses which to admit in `allowedCommitRequests`. + +Costs: + +- **A request outside every lent namespace is seen by nobody and gets no status.** +- **The watches change when a provider grant changes.** Grants change rarely, and the mirror + already manages watches per namespace. +- **Watching only the namespaces the GitTarget mirrors would be narrower, and was not chosen.** It + would change with every WatchRule edit, and it would lose `NotAllowed` for namespaces that are + lent but not mirrored. + +#### Not chosen + +- **An explicit namespace list on the provider**, `commitRequests: {mode: Reconcile, + sourceNamespaces: [...]}`. It would be a third list of namespaces, next to `allowedGitTargets` + and `allowedCommitRequests`, and the three would drift apart. +- **One connection with impersonated identities**, the way an Argo CD AppProject uses + `destinationServiceAccounts`. `impersonate` is close to cluster-admin, and it concentrates power + in one credential, which is the opposite of what several identities are for. +- **Splitting the cluster from the identity, and riding the GitTarget's watch streams.** These are + in [Future work](#future-work). + +### cert-manager + +You were right: it does not go away. The audit ingress mints three things through cert-manager: + +- a root CA; +- its serving certificate; +- the kube-apiserver client certificate for mTLS (`audit-certificates.yaml`). + +These render only with `attribution.enabled`. Metrics TLS is off by default. So once admission is +removed: + +- **Default install** (no attribution): no certificate is rendered, and cert-manager is not + needed. +- **Attribution on**, which includes the managed product: cert-manager is needed, or Secrets you + bring yourself with `servers.audit.tls.certManager: false`. + +Two chart changes make this true: + +1. Gate the self-signed `Issuer` in `certificates.yaml` on some certificate being rendered, meaning + attribution with audit TLS, or metrics TLS. Today it renders whenever `certManager.enabled` is + true, which is the default, so an install without cert-manager's CRDs would fail on it. +2. Move [quickstart.md](../quickstart.md) step 1 into the "Want named users on your commits?" + section. + +## Future work + +### Future work: creating the CRD + +Not in this design. If it comes, these are the rules: + +- **Create it only when it is absent.** While either `Reconcile` value is set and discovery shows + that the cluster serves no `commitrequests.configbutler.ai`, the CRD is created from a copy built + into the operator binary. `AlreadyExists` counts as present. `NotInstalled` would then be + replaced by `Created` (True) and `CreationFailed` (False, with the API error and the missing + permission). +- **Never update it, and never delete it.** Not after an operator upgrade, and not on `Ignore` or + when the KubeProvider goes. Deleting a CRD deletes every CommitRequest on the cluster. + +Updating is ruled out for good, because it is a minefield: + +- **An update changes the rules for objects that are already stored.** A version that stops being + served, or validation that gets stricter, makes the next status write fail on an object the + update broke. +- **The CRD may have another owner**: Helm's `crds/`, Flux or Argo CD applying it from Git, + gitops-api's `workspaceCRDs`, or a kcp `APIExport`. Two writers fight, and a GitOps owner reverts + the operator's change. +- **An operator upgrade would change the API on every connected cluster at once**, with nobody + reviewing it. +- **The permission is broad.** `update` on CRDs is power over the cluster's API surface. + +Creating is different, because it replaces nothing. Its cost is the permission: `create` on +`customresourcedefinitions` is cluster-wide, on a credential that is otherwise read-mostly. Its +gain is one step fewer for every new source cluster or kcp workspace. In kcp, an `APIBinding` that +already provides `configbutler.ai` shows up in discovery, so nothing would be created. That is the +right outcome, but it would need a test. + +### Future work: other items + +- **A cluster ID** per KubeProvider, such as the `kube-system` namespace UID, or the logical cluster + name in kcp. It would close the known gap in + [Which path serves a request](#which-path-serves-a-request): the operator could tell that a + kubeconfig points back at the control cluster. +- **Split the cluster from the identity.** A new kind, for example `KubeCluster`, would hold the + endpoint, the audit route, the CRD state and the cluster ID. A KubeProvider would become one + identity on a cluster. + - It fixes `AuditRouteConflict` across namespaces, because every identity on a cluster should + read that cluster's one audit stream. + - It also stops N providers from reporting the same CRD state. + - It costs a new kind, a breaking split of KubeProvider, and a platform-team owner for the + cluster object. + - kcp gives every workspace its own logical cluster with one identity, so the managed product + does not need it. Build it when the first `AuditRouteConflict` between two identities shows up. +- **CommitRequests ride the GitTarget's own watch streams.** The request's author would come from + the same inline `attachAuthor` call that attaches an edit's author, and the source-cluster path's + own server would go away. The watch plane is the most delicate code in the project, so this is + too large for now. + +## What exists today + +| | `SaveRequest` (`configbutler.ai/v1alpha3`) | `RemoteSaveRequest` (`source.configbutler.ai/v1alpha1`) | +| --- | --- | --- | +| Created on | the control cluster | the source cluster (kcp workspace, or any KubeProvider) | +| Author from | the `/validate-operator-types` admission webhook, written to Redis `author:v1:command:` | the source cluster's audit facts, same index as the edits | +| Author when that source is absent | none (`AuthorAttributed=False`). The default install is in this state: admission on, Redis empty | none, and refused `AuthorUnresolved` if the provider ever sent audit | +| Admitted by | `GitTarget.spec.allowedSaveRequests` (default: `from: Same`) | `GitTarget.spec.allowedRemoteSaveRequests` (default: none) and `KubeProvider.spec.remoteSaveRequests: Enabled` | +| Window spec | `attach` (`CurrentOrNext`/`Next`), `author` (`Own`/`SavedBy`/`TakeOver`), `attachTimeout` 2s, `idleTimeout`, `maxDuration` 2s | `attachTimeout` 5s, `idleTimeout`, `maxDuration` 2s | +| Extra spec | `whenNothingToCommit` (`Resolve`/`CommitEmpty`), `catchUp` | `catchUp` | +| Served by | a controller-runtime reconciler: `saverequest_controller.go`, `_finalize.go`, `_apply.go`, 1,468 lines | `internal/watch/remote_save_requests.go`, 702 lines | +| Same worker API | `git.AttachSaveRequest` | `git.AttachSaveRequest` | + +The decided design keeps both ways of picking a request up, under the left-hand column's name and +API group. It has one core with the right-hand column's identity rule, and one grant list. Both +end up in the same branch-worker code (`internal/git/save_request_attach*.go`, 1,212 lines), which +stays. + +## How it got here + +The design history, in the steps that matter here: + +1. **Audit with a 60s wait** (`c6220afc`, May). The `CommitRequest` author was read from its own + create event in the ordered audit stream. It flaked on `generateName` when an event was diverted + below the high-water mark (`91cc3172`). +2. **Admission** (`58dd37a8`, `96ec3906`, June). It was introduced for two reasons. Option C "kept + the wrong source … and therefore kept a wait", and "on a managed cluster where you cannot enable + an apiserver audit webhook, your commands are still attributed". +3. **The keyed fact index** (#211, July) and **watch-first ingestion** removed both reasons. + - A fact is now awaited from the index, bounded by `--author-attribution-grace` (3s by default, + `internal/watch/author_resolver.go:26`). + - That is the same pipeline, and the same wait, as the edits the save publishes. + - The 60s wait and the ordered-stream divert are gone. +4. **The race admission created** (#367, September). The request took the fast path (admission) and + its edit the slow one (audit join). Saves were "finalized before the write it exists to publish + reached the worker", so `attachTimeout` got a 2s default to cover the gap + ([wait-options](../design/commitrequest-save-wait-options.md), l.16-43, 97-114). +5. **"Works without audit" was never true on its own.** A named request on an audit-less cluster + meets windows that name nobody: `WindowMismatch`, "every time, permanently" + (wait-options l.277-281). It became true only on 2026-10-26, with `SavedBy`/`TakeOver` + (`cf3b97be`, `ee8946c5`), and only with an opt-in grant and an un-latched provider. +6. **`RemoteSaveRequest`** (`beddcdef` to `70d07d91`). It is authored from source audit. It was + built so that "a customer's only API surface is their kcp workspace" holds with no exceptions: + no tenant grant into the control cluster, and no `/editor/commit` on the save path + ([source-side-save-plan](../future/source-side-save-plan.md), l.46-59). + +In short: admission was the right answer to a pipeline that no longer exists. Every argument for it +since then has been about either takeover or gitops-api's current save path, and the source-side +plan already replaces that path. + +## Review findings + +F1 to F10 are complexity findings, not correctness bugs; the correctness review of these commits +happened in the three rounds recorded on the PR. F11 is a correctness bug, found while working out +the design. + +**F1. The default install pays for admission and gets nothing for saves.** The chart has +`servers.admission.enabled: true` while `queue.redis.addr` is `""`. The command-author store is +nil, so every `SaveRequest` reports `AuthorCaptureDisabled`. The webhook's only effect in the +default install is the WatchRule feedback (F6). + +**F2. cert-manager is a prerequisite of the default install only because of admission.** + +- The admission certificate is the only certificate a default install renders, and + [quickstart.md](../quickstart.md) step 1 says so: "The controller mounts an admission certificate + at startup, so cert-manager must be healthy before…". +- Attribution still needs cert-manager (see [cert-manager](#cert-manager)). +- [where-validation-lives](../spec/where-validation-lives.md) prices a webhook as "cert wiring in + the chart, a failure mode that can block tenant writes, and an extra moving part in every install + path". + +**F3. Two identity domains, joined only by name equality.** A control-cluster save attaches to a +window only when the admission `userInfo.username` equals the audit user on the source cluster. +That holds for gitops-api only because its server impersonates the Dex email +(source-side-save-plan l.42-48). For anyone else with two clusters, it is a coincidence. The design +removes it by refusing a control-cluster request whose GitTarget mirrors another cluster. + +**F4. Two servers for one job, already drifting.** + +- The reconciler has these, and the remote server lacks them: + - `save_requests_total`; + - the phase reasons (`WaitingForWorker` / `WaitingForWindow` / `CollectingWindow` / + `WaitingForPush`); + - `attach: Next`; + - `CommitEmpty`; + - the `AuthorAttributed` condition. +- The remote server has these, and the reconciler lacks them: + - `status.author`; + - `AuthorUnresolved`; + - a 5s default instead of 2s, because its timing model differs. +- `gitTargetUID` handling exists in both, written twice: `refuseOnceUnheld` / `readmit`. +- Every later fix has to be made twice, or the two drift further. The design keeps two ways of + picking a request up, but with one core behind them. + +**F5. Takeover is the largest piece of policy on the branch, and it serves one case.** + +- It takes three conditions, two enums of three values, the provider latch, a per-grant bound, + "max over admitting grants", and two commit trailers (`ee8946c5` + `cf3b97be`: +767/−171 lines). +- It exists only for a control-cluster save on a source cluster with no audit. +- Its name can come only from admission (source-side-save-plan l.260-262), so it is the one feature + that makes admission load-bearing. + +**F6. The WatchRule check on the same endpoint is feedback, not a gate.** + +- The writer refuses the configuration whether or not the webhook ran. +- The GitTarget already reports it as `MultipleSourceNamespaces` + (`internal/controller/gittarget_controller.go:91`). +- `failurePolicy: Ignore` means the webhook was never the guarantee. + +**F7. `/validate-all` is dead weight in e2e.** + +- It always allows, and matches every write on every resource with `timeoutSeconds: 1` + (`config/webhook/validating-webhook.yaml`). +- The chart does not ship it. +- Every API write in the e2e system under test makes an extra webhook round trip for nothing. + +**F8. No e2e proves a save's author round trip on the audit path.** + +- The kcp corner runs without audit (`test/e2e/setup/kcp/instance.yaml:11-14`), so + `RemoteSaveRequest` is only exercised on the unnamed-attach path + (`test/e2e/source_cluster_e2e_test.go:872`). +- The main suite runs with audit on, against the in-cluster provider. Once the control-cluster path + takes its author from audit instead of admission, `save_request_e2e_test.go` proves the round + trip with no change to where its requests are created. + +**F9. Two grant lists with opposite defaults on one GitTarget.** + +- `allowedSaveRequests` defaults to `from: Same`, with no `omitempty`, a Go mirror and an envtest. +- `allowedRemoteSaveRequests` defaults to none. +- A reader has to learn both, and why they differ. The design has one list and one default. + +**F10. Docs gap.** + +- [security-model.md](../security-model.md) does not list the admission endpoint or the + command-author Redis records as a trust boundary, and its Redis row says "Buffers decoded audit + events". +- The gap goes away with the endpoint. + +**F11. Two KubeProviders on one cluster fight over a request.** + +- `RemoteSaveRequestServer.admit` returns `NotAllowed`, "GitTarget … mirrors another cluster than + this one", when the request's GitTarget names a different KubeProvider + (`internal/watch/remote_save_requests.go:480`). +- `serve` writes that as a final `Ready=False`, and a finished `Ready` is never served again + (l.353). +- Two providers enabled on one cluster therefore race: for example the chart's `local` provider + next to a team's own `inCluster` one, or two kubeconfigs with different credentials. If the + provider that does not serve the GitTarget sees the request first, the request fails. +- It also waits for the author before it checks the GitTarget, so the wrong provider spends the + attribution grace first. +- The design fixes it, because the GitTarget's provider alone decides who serves + ([Which path serves a request](#which-path-serves-a-request)). + +## Options considered + +- **A: remove admission and takeover, keep both kinds.** Not chosen. Two kinds would keep F4 and + F9. +- **B: one kind on the source cluster only**, in the first version of this review named + `SaveRequest` in `source.configbutler.ai`. It was chosen at first and later revised. It made + CommitRequest the only operator kind that a single-cluster user reaches through a KubeProvider, + with a provider switch, a grant in source-cluster terms, a derived watch scope and CRD conditions + on the `local` provider. +- **B': one kind, two ways of picking it up, one core.** **Chosen.** A single-cluster user gets the + behavior of `main`, and other clusters get the special path. +- **C: no save kind at all.** Not chosen. It removes the user's message on their commit and the + `catchUp: AfterPush` signal built for gitops-api. + +## What goes, by numbers + +Line counts are `wc -l` at `70d07d91`. Rows marked "≈" are portions of shared files, estimated from +the functions and blocks involved. + +### Admission and takeover + +| Item | Non-test | Test | +| --- | ---: | ---: | +| `validate_operator_types_handler.go`, `admission_allow_handler.go`, `watchrule_source_namespace_admission.go` | 339 | 390 | +| `internal/queue/command_author_store.go` | 84 | 83 | +| `saverequest_attribution_agreement_test.go` | | 152 | +| `cmd/main.go`: flags, validation, server setup, store wiring (48 lines mention admission) | ≈110 | ≈40 | +| Takeover: API enums, grant field, `claimOn`, `claims`, trailers, `openWindow.claim` | ≈250 | ≈280 | +| Chart: `admission-server-certificate.yaml`, `validate-operator-types-webhook.yaml`, values, deployment, services, helpers | ≈150 | | +| `config/`: `webhook/` (124), admission `Certificate`, deployment, service and kustomization lines | ≈170 | | +| Docs: `spec/saverequest-admission-authorship.md` (68), sections in where-validation-lives, quickstart step 1, chart README rows | ≈150 | | +| **Subtotal** | **≈1,450** | **≈950** | + +### One kind, one core + +| Item | Non-test | Test | +| --- | ---: | ---: | +| The `source.configbutler.ai` group: `api/source/v1alpha1` (411) and its CRD (299), plus scheme, PROJECT and kustomize lines | ≈740 | | +| The control-cluster reconciler cut down to a front end over the shared core: 1,468 lines become about 300 | ≈1,170 | ≈1,200 | +| `requestSelector`, and the second grant list merged into one | ≈80 | ≈100 | +| RBAC rules, deepcopy, `hack/crdfields` rows | ≈80 | | +| Added back: F11's dispatch rule, the skipped await, the `NotAllowed` message for a GitTarget that mirrors another cluster | ≈−150 | ≈−250 | +| Added back: the revision check, both KubeProvider conditions, the derived watch scope | ≈−250 | ≈−350 | +| **Subtotal** | **≈1,670** | **≈700** | + +**Total: about 3,100 non-test lines (about 700 of them generated CRD and YAML) and about 1,650 +test lines, roughly 4,800 net.** + +- This is less than option B's 6,600, because the control-cluster path stays. That buys the + behavior of `main` for single-cluster users. +- The `v1alpha3` `CommitRequest` type and CRD stay, with the remote kind's spec and status. +- The rename of `b86dffbe` is undone for the kind. That is a large diff but adds no lines. +- `save_request_e2e_test.go` (620 lines) stays where it is. Only its kind name and its assertions + on the author change. + +### Kept: `attach: Next` and `whenNothingToCommit: CommitEmpty` + +Both stay in the design (decided 2026-10-27). They move into the shared core, so the source-cluster +path gains them, and the counts above are unchanged. The source-cluster server never had them, so +the move needs tests on that path: `Next` skipping an open window, and `CommitEmpty` making a +commit when there is nothing to save. + +### Settings + +| Kind | Removed | Added | +| --- | --- | --- | +| Operator flags | `--admission-webhook`, `--admission-webhook-bind-address`, `--admission-webhook-cert-path`, `--admission-webhook-cert-name`, `--admission-webhook-cert-key` | | +| Chart values | `servers.admission.{enabled, bindAddress, port, timeoutSeconds, tls.certManager, tls.certPath, tls.certName, tls.certKey, tls.secretNameOverride}` | | +| Install prerequisites | cert-manager for the default install (still needed with attribution) | | +| Kubernetes objects | `ValidatingWebhookConfiguration`, the admission `Certificate` and its Secret, Service port 9443, the `source.configbutler.ai` API group and its CRD | | +| Redis | `author:v1:command:*` | | +| KubeProvider | `spec.remoteSaveRequests` | `spec.commitRequests`, not with `inCluster`; condition `CommitRequestsServed`, printer column `Saves`, `status.commitRequests.namespaces`; in phase 2, condition `CommitRequestCRDUpToDate` | +| GitTarget | `spec.allowedSaveRequests`, `spec.allowedRemoteSaveRequests`, `unattributedWindows`, `requestSelector` | `spec.allowedCommitRequests`, whose grants set `consumerNamespaces` or `sourceNamespaces` | +| CommitRequest | `spec.window.author` | `gitTargetRef.namespace` (optional; required on a source cluster, where a request without it is not picked up), `catchUp` and `status.author` | +| CRD | | the schema revision line in the kind's description, and its hash test | +| Release assets | | `commitrequest-crd.yaml` | +| Conditions and reasons | `AuthorAttributed` (`AttributedFromAdmission`, `CommitterFallback`, `AuthorCaptureDisabled`), `TakeOverNotAllowed` | | +| Commit trailers | `Saved-By:`, `Author-Source:` | | + +Compared with `main`, which has `CommitRequest` and the admission webhook, a single-cluster user +sees only these changes: + +- the request may name a GitTarget in another namespace, which then admits it in + `allowedCommitRequests`; +- the author comes from audit, not admission; +- the webhook is gone, and with it the need for cert-manager when attribution is off. + +Saving on another cluster is new. + +## If takeover is needed again later + +Do not bring back a webhook server for it. Two lighter routes exist, and neither is proposed now: + +- **A `ValidatingAdmissionPolicy` on the source cluster** that requires + `object.spec.savedBy == request.userInfo.username`. + - It is in-tree CEL, GA since Kubernetes 1.30, with no server, certificate or Redis. + - kcp appears to support it per workspace (kcp PR #3743). That is unverified for the release you + run. + - It collides with the declined #5 principle, "authorship is established, never asserted" + ([attribution.md](../spec/attribution.md) l.39-67): the operator cannot prove the policy ran + when the request was created. That objection would have to be overruled knowingly. +- **A `MutatingAdmissionPolicy`** that stamps the creator onto the object. It is beta in 1.34 and + tracked for GA in 1.36, and the same objection applies. + +## What this frees time for: the kcp guide + +The pitch fits in one sentence: edit a data-only cluster, and every edit and every save lands in +Git under the name of the person who made it. + +**There today:** + +- a kcp install as kcp-operator CRs (`test/e2e/setup/kcp/`); +- a kubeconfig per workspace through the front-proxy (`test/e2e/kcp_workspace_test.go`); +- `kcp.io/cluster` stripped from Git; +- `attribution.auditRouteAnnotationKey`, for one shared audit stream. + +**Missing:** + +- the shard audit flags, and the annotation that routes each event to a workspace's KubeProvider; +- per-workspace OIDC (`WorkspaceAuthentication`), so audit names a person rather than an admin; +- installing `commitrequest-crd.yaml` into each workspace, which gitops-api's `workspaceCRDs` + already does in the managed product; +- the JSON-only caveat; +- an e2e with kcp audit on, which would also close F8 for the source-cluster path. + +The e2e instance leaves each of these out on purpose (`instance.yaml:11-14`), and the reference +deployment has them. + +**Order:** an e2e kcp corner with audit on comes first. A test then keeps the guide true, and it +is the first place a save's audit round trip is proven across clusters. + +## Proposed sequence + +Hard cuts, no migration. + +1. **gitops-api switches to the source-side save**: the browser creates a `CommitRequest` in the + workspace. This is the only ordering constraint. Until the cut lands, the kind it creates is + `RemoteSaveRequest`, so gitops-api can also wait and switch once, straight to the final name. +2. **One PR implements the design:** + - Delete the admission server, the command-author store, the takeover and the + `source.configbutler.ai` group. + - Rename the kind back to `CommitRequest`. + - Extract the shared core. The control-cluster reconciler and the source-cluster server become + front ends over it, with the dispatch rule that fixes F11. + - Add `allowedCommitRequests` with its two fields, and `commitRequests` with its CEL rule. + - In the core: acquire the fact stream for `commitrequests`, make status writes conditional on + the resourceVersion with a fresh decision on conflict, claim the request before attaching + (rule 3), and stamp `delete-after` for retention. + - Move `attach: Next` and `whenNothingToCommit: CommitEmpty` into the core, so both paths + have them. + - Scope each provider's watch to the namespaces it lends, with `status.commitRequests.namespaces` + and the `CommitRequestsServed` condition. + - Gate the chart `Issuer`, and move the quickstart's cert-manager step under attribution. + - Rewrite [commitrequest-design.md](../spec/commitrequest-design.md) to describe the one kind, and + add a sentence to [definitions.md](../definitions.md): CommitRequest is the one operator kind + that can also live on a source cluster. + - Publish the `commitrequest-crd.yaml` release asset, with the schema revision line and its hash + test. + - Tests, described in [Acceptance](#acceptance). +3. **Phase 2 of the CRD signal**, as its own PR: first verify that kcp serves per-workspace OpenAPI + v3, then add the `CommitRequestCRDUpToDate` condition, with a unit test per reason. +4. **The kcp audit e2e corner, then the guide.** + +## Acceptance + +The main PR is done when these pass, besides `task lint`, `task test` and `task test-e2e`: + +- **Overlapping observers**, as unit tests with fake clients: + - a control-cluster request seen by the local controller and by a kubeconfig watcher pointing at + the same cluster, with the namespace omitted and with it set; + - two kubeconfig providers on one source cluster; + - a GitTarget created during the attach wait, in both orders: when the owner's claim wins, the + non-owner cannot overwrite it; when the non-owner's `GitTargetNotFound` wins, the owner neither + attaches nor commits; + - a status conflict after the worker has produced an outcome: the outcome is re-applied, and the + save is not restarted; + - a restart while a request is `Saving`: the owner resumes it. +- **The existing source-cluster tests keep passing** through the extraction of the core, including + the kcp corner in `test/e2e/source_cluster_e2e_test.go`. +- **`attach: Next` and `CommitEmpty` on the source-cluster path**, which has never had them: `Next` + skips an open window and takes the next one, and `CommitEmpty` makes a commit when there is + nothing to save. +- **Delayed audit delivery**, as a unit test: the request's fact arrives late, but within the + grace, on a provider whose `AuditFactsReceived` is not yet True. The request is attributed. +- **Retention**: `delete-after` is stamped and honored on both paths. Removing the annotation keeps + the request. +- **The main e2e suite** asserts that a save's author comes from audit on the control-cluster + path. That is the round trip admission used to stand in for. + +**The kcp audit e2e is not a gate for removing admission.** The source-cluster path never used +admission, and removing admission changes only the control-cluster path, which the main suite +covers with audit on. The kcp audit corner proves the source-cluster round trip, and stays step 5. + +## Review round 2 (2026-10-27) + +An outside review of the previous revision raised five points: + +| Point | Verdict | Change | +| --- | --- | --- | +| A source watcher can refuse a valid control-cluster request | **Accepted.** The scenario needs a kubeconfig that points back at the control cluster. That is not an exotic case: it is how a tenant who is not listed in `--in-cluster-kube-providers` gets its own identity there | `GitTargetNamespaceRequired` is gone, and rule 1 applies: a path writes only what it knows to be true | +| Skipping the author wait before the first audit fact loses authors | **Accepted.** The shortcut was this design's own addition, and the code checks `sentAuditFacts` after the wait. The fix is to not add the shortcut | the wait runs whenever attribution is on | +| Several writers of `GitTargetNotFound` are unsafe | **Accepted, narrower than stated.** It needs two observers of one cluster and a GitTarget created during the attach wait. The cause exists, though: the writer overwrites without deciding again | rule 2: writes are conditional on the resourceVersion, and a conflict means deciding again | +| Adopting the remote TTL breaks `delete-after` retention | **Accepted** | the core keeps the annotation contract on both paths | +| The CRD compatibility protocol is large and unresolved | **Partly accepted.** The status writer does erase fields a newer schema added, and the fallback to reading the CRD cannot see an API that an `APIBinding` provides. Deferring the signal entirely was **not** accepted, because an outdated CRD is a signal you asked for | split into phase 1, discovery only, in the main PR, and phase 2, the revision, as its own PR behind a kcp check. A newer CRD is never served, so no preservation rules are needed | + +Also accepted: the core's responsibilities are now listed, including acquiring the fact stream, +which the control-cluster reconciler never did. + +**Rejected: the kcp audit e2e as a gate for removing admission.** See [Acceptance](#acceptance). + +## Review round 3 (2026-10-27) + +The same reviewer approved phase 1, and raised one invariant and four smaller points: + +| Point | Verdict | Change | +| --- | --- | --- | +| The owner must persist its claim before attaching, because today's `serve` continues after a failed write | **Accepted.** Checked: `write` only logs | rule 3, and both race orders in [Acceptance](#acceptance) | +| Deciding again after a later conflict must not restart a save, and ownership must survive a restart | **Accepted** | under rule 3 | +| The RBAC list omitted `patch` | **Accepted** | listed per mode | +| Not every condition message should say to apply the CRD | **Accepted** | a fix per reason | +| A missing revision means "compatibility unknown", not "from elsewhere" | **Accepted, and taken further.** Phase 1's release asset would otherwise ship without a revision, so every cluster set up in phase 1 would be unjudgeable in phase 2 | the revision line ships in phase 1; phase 2 reports `RevisionUnknown` and keeps serving | +| Keep `Next` and `CommitEmpty` unless their removal is settled | **Accepted**, and since settled: both stay | ported into the core | diff --git a/docs/security-model.md b/docs/security-model.md index 20ad641cc..6d3e33e7d 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -47,8 +47,13 @@ To mirror live state into Git the controller must: or `applierAccess.argocd`. Apart from the applier an applier claim holds, the controller never writes to a watched type. Its -write targets are Git, that applier, and the Secrets it generates itself (the signing key, and the -age key under `generateWhenMissing`). +write targets are Git, that applier, the Secrets it generates itself (the signing key, and the +age key under `generateWhenMissing`), and the status of the `CommitRequest`s it serves. On a source +cluster that last write is opt-in: a `KubeProvider` with `spec.commitRequests` set writes +`CommitRequest` status there, and, with `ReconcileAndDeleteFinished`, deletes finished requests, +through its own credential, in the namespaces it lends. It never writes the CRD. + +GitOps Reverser installs no admission webhook, so it never sits in the write path of the API server. ### Which namespaces a rule may read from @@ -113,13 +118,14 @@ namespace it already knows. Narrowing that further is |---|---| | Git credentials Secret | Grants push access to your repository. | | SOPS/age key material | Decrypts (and the public key encrypts) Secret data written to Git. | -| Redis/Valkey queue | Buffers decoded audit events in transit; not an audit archive. | +| Redis/Valkey queue | Holds each `GitTarget`'s watch resume cursors, and, with the `redis` attribution transport, the attribution facts decoded from audit events, keyed by audit route, type and object, until their TTL; not an audit archive. | | Audit ingress (`/audit-webhook/`, such as `/audit-webhook/gitops-reverser.local` for the chart's provider) | Accepts audit traffic; protected by mutual TLS via cert-manager. A route is held by one namespace: a `KubeProvider` elsewhere that carries it is `Stalled` (`AuditRouteConflict`) and reads none of its facts. | | Push receiver (`--receiver-bind-address`, off by default) | Plain HTTP for Git hosts' push webhooks. Each `GitReceiver` route is a path derived from its token, whose Secret lives in the receiver's own namespace, and a delivery must carry a valid signature or token for it before its body is read. A delivery is only a hint: it wakes the branch worker of every `GitProvider`, in any namespace, whose `spec.url` names the repository it reports, and that worker reads the remote with its own provider's credential. The payload's repository URL is only compared with `spec.url`, never fetched, so a delivery cannot choose a URL, a credential or a commit. A worker holds at most one pending look however many deliveries arrive, so deliveries never queue up behind its writes; but they are not rate-limited, so a token holder, in whatever namespace, who keeps delivering can keep a matching worker reading its remote continuously. That availability cost is accepted: there is no per-worker cooldown. The `202` carries no body, so it does not say whether another namespace tracks a repository. | | Generated Secret material | Signing keys and generated age keys live in cluster Secrets. | | Grants (`allowedGitBranches`, `allowedGitTargets`, `allowedApplierClaims`) | Nothing is admitted automatically, an object's own namespace included. A `from: Selector` grant admits every control-cluster namespace whose labels match, so it is only as strong as the right to set those labels: whoever can label a namespace can admit it, including a tenant allowed to edit its own namespace. Select on labels only the platform can set, or use `from: Names`. A `Names` grant also admits a namespace deleted and recreated with the same name. Grants decide who may use a provider's identity; every operation still runs as that identity, never as the referrer. | | Another provider on the same repository | The branch claim is keyed by repository and branch, not by provider, so grants cannot stop a `GitBranch` that claims a branch through a second `GitProvider` for the same URL with a copied credential. On Gitea and Forgejo a read-only token opens a push session. Keep tenants from creating `GitProvider`s where they share a repository. | | `KubeProvider.spec.inCluster` and `--in-cluster-kube-providers` | `inCluster` wields the operator's own identity in the cluster it runs in. It is honored only for a `KubeProvider` the flag lists by exact `namespace/name` (no wildcards, empty by default); any other is `Stalled` (`InClusterNotAuthorized`) and never used. Anyone who can create a `KubeProvider` under a listed name wields that identity through it, so list only names in namespaces you control. | +| `GitTarget.spec.allowedCommitRequests` | Lets whoever may create a `CommitRequest` in an admitted namespace save their own commit window on the `GitTarget`: its message, its timers and an apply. `consumerNamespaces` admit namespaces of the cluster GitOps Reverser runs in; `sourceNamespaces` admit namespaces of the cluster the `GitTarget` mirrors, by name only, and never one its `KubeProvider` does not lend it. A request's author comes from its own audit event, never from a name it carries, so a grant decides who may save, not who authored the commit. | | `KubeProvider.spec.allowedApplierClaims` | Lets the namespaces a grant admits suspend and apply a `Kustomization` or `Application` in its `applierNamespaces`, such as `flux-system`, through the provider's identity. Admission to mirror does not admit to apply. A delegation, not owner consent: it does not prove the namespace owns the applier or its repository. An applier claim only acts while an eligible `GitTarget` names it, and the binding check compares the `GitProvider` URL, not write access. A denial hands the applier back. | ## Secret data the controller writes to Git diff --git a/docs/spec/README.md b/docs/spec/README.md index b1fc6b72d..7ca326264 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -47,7 +47,6 @@ three lifecycle classes and was unreadable spread across them. The rows below st | [`scale-subresource-audit-rehydration.md`](scale-subresource-audit-rehydration.md) | history: the `/scale` field-patch design; a scale is now mirrored from the watch, and the field-patch writer is what remains | | [`commit-window-refactor.md`](commit-window-refactor.md) | one grouped commit = one (author, GitTarget) | | [`commitrequest-design.md`](commitrequest-design.md) | how a request binds to a same-actor commit window and reports its outcome | -| [`commitrequest-admission-authorship.md`](commitrequest-admission-authorship.md) | a request's submitter comes from admission, present-or-never, apart from audit facts | | [`status-conditions-guide.md`](status-conditions-guide.md) | the conditions and reasons every kind reports | | [`gitpath-foreign-content-stringency.md`](gitpath-foreign-content-stringency.md) | a `GitTarget` owns its subtree and refuses foreign content; `.gittargetignore` | | [`unsupported-folder-refusal-plan.md`](unsupported-folder-refusal-plan.md) | history of the `GitPathAccepted` refusal of unsupported folder content | diff --git a/docs/spec/attribution.md b/docs/spec/attribution.md index 9b2b27a41..65985b1d8 100644 --- a/docs/spec/attribution.md +++ b/docs/spec/attribution.md @@ -18,7 +18,6 @@ > [what attribution gives up, on purpose](#what-attribution-gives-up-on-purpose). > > Related: [`watch-event-ordering-and-attribution-grace.md`](watch-event-ordering-and-attribution-grace.md), -> [`commitrequest-admission-authorship.md`](commitrequest-admission-authorship.md), > [`../attribution-setup-guide.md`](../attribution-setup-guide.md), > [`../interpreting-metrics.md`](../interpreting-metrics.md). @@ -63,8 +62,8 @@ That is why `CommitRequest` has no `spec.author`, although one was asked for (co To commit under your own name, authenticate as yourself. Use your identity provider (OIDC, for example) so that the API server knows who you are. A request that cannot be attributed claims no -actor (`AuthorAttributed=False`); it never falls back to a name it was given. See -[`commitrequest-admission-authorship.md`](commitrequest-admission-authorship.md). +actor; it never falls back to a name it was given. A `CommitRequest`'s own author comes from its +audit event the same way, in [`commitrequest-design.md`](commitrequest-design.md). ## 1. Deletion is attributed at intent time diff --git a/docs/spec/commitrequest-admission-authorship.md b/docs/spec/commitrequest-admission-authorship.md deleted file mode 100644 index 9818f09f6..000000000 --- a/docs/spec/commitrequest-admission-authorship.md +++ /dev/null @@ -1,68 +0,0 @@ -# CommitRequest authorship from admission - -> **spec** — current behaviour. The code depends on this document; change one, change the other. -> Index: [`../INDEX.md`](../INDEX.md) - -A `CommitRequest` is a command to close an existing GitTarget commit window. Its submitter is captured -by the `/validate-operator-types` validating admission webhook, while mirrored-resource attribution is -separately derived from kube-apiserver audit facts. The two paths answer different questions and must not -be conflated. - -## Contract - -| Concern | CommitRequest submitter | Mirrored-resource author | -|---|---|---| -| Source | validating admission request `userInfo` | post-persist audit fact joined to watch event | -| Key | CommitRequest UID | source provider, GVR, UID, and resourceVersion | -| Timing | written before the object persists | waits up to `--author-attribution-grace` after the watch event | -| Miss | request claims no actor | watch commit has the explicit unresolved author | - -The admission handler records the authenticated submitter in the Redis command-author store and always -allows the request. It is deliberately best-effort: a Redis failure or an unavailable webhook must not -reject a user's command. The controller reads the record when the persisted `CommitRequest` is first -reconciled. The result is **present-or-never**; waiting cannot create a record that admission did not write. - -## Attach and final Git author - -`AuthorAttributed=True` means admission captured the command submitter. The request can attach only to an -open window with that same named actor and GitTarget. - -`AuthorAttributed=False` (`CommitterFallback`) means capture ran but no admission record was available; -`AuthorCaptureDisabled` means capture was not configured. Neither is a failure and neither means the -eventual Git author is necessarily the configured committer. The request claims no actor and can attach only -to an unnamed window. That window determines the actual Git author: - -- configured-author mode or a replay/resync write: the configured committer; -- live attribution that ran but found no usable audit fact: - `unknown (attribution unresolved) `. - -The worker never closes another actor's window. If no matching window appears before `attachTimeout` -runs out, the request ends successfully with `Ready=True` and `NoWindow` or `WindowMismatch`. That is -`Pushed=False`, except for a `NoWindow` request with `whenNothingToCommit: CommitEmpty`, which records -its message in a pushed empty commit (`Pushed=True`). A `WindowMismatch` never falls back to one. - -## Conditions - -`AuthorAttributed` is binary and settled on the first reconcile; it has no audit-wait or `Unknown` state. - -| Condition | True | False | -|---|---|---| -| `AuthorAttributed` | `AttributedFromAdmission`: the command submitter was captured | `CommitterFallback`: capture ran but no command-author record; `AuthorCaptureDisabled`: capture is off; request claims no actor | -| `Pushed` | the attached window was committed and pushed | a benign no-commit or finalize failure | -| `Ready` | a pushed commit or benign no-commit | progress or a finalize failure | - -`Reconciling=True` is the normal in-progress state, with the phase the branch worker reports as its -reason: `Progressing` until the worker registers the request (`WaitingForWorker` while the GitTarget -has no branch worker yet), then `WaitingForWindow`, `CollectingWindow` and `WaitingForPush`. - -## Wiring - -The command-author store is wired when the admission webhook is enabled and Redis is configured. It is -independent of `--author-attribution`, which controls audit-backed authorship for live watched resources. -An installation may therefore have either, both, or neither source of author information. - -Related live contracts: - -- [architecture: CommitRequest Finalize](../architecture.md#commitrequest-finalize) -- [configuration: CommitRequest conditions](../configuration.md#when-there-is-nothing-to-commit) -- [status conditions guide](status-conditions-guide.md) diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index 46896136d..130f54d90 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -1,6 +1,6 @@ # CommitRequest window finalization -> **spec**: current behavior, reviewed at `c12f7970` on 2026-10-20. +> **spec**: current behavior, reviewed at `670032d2` on 2026-10-27. > The code depends on this document; change one, change the other. > Index: [`../INDEX.md`](../INDEX.md) @@ -8,11 +8,84 @@ A `CommitRequest` is a one-shot “save now” command for one `GitTarget`. It d `CommitRequest` object. Instead, it can attach a message to a matching open commit window and asks the worker to close that window after the requested collect delay. +## Two ways in, one core + +A `CommitRequest` is the one operator kind that can also live on a source cluster. It is picked up +in one of two ways, and both hand it to one core +([`commitrequest_core.go`](../../internal/controller/commitrequest_core.go)), which decides everything +else once: + +- **On the control cluster**, the `CommitRequestReconciler` serves it with the operator's own client, + as any operator kind. +- **On a source cluster**, the `CommitRequestServer` watches the requests of every `KubeProvider` + whose `spec.commitRequests` is `Reconcile` or `ReconcileAndDeleteFinished`, through that provider's + credential, in the namespaces it lends its `GitTarget`s in `spec.allowedGitTargets`. It writes + status, and with `ReconcileAndDeleteFinished` deletes finished requests, through the same + credential. A rotated kubeconfig restarts the watch on a client built from the new credential, so + watches and status writes never keep the old one. It never writes the CRD, and reports its state in + the `KubeProvider`'s `CommitRequestsServed` condition. + +The core takes one step at a time, each decided again from the object as it is stored. Its +responsibilities, in order: admission by `spec.allowedCommitRequests`; the request's author; the +attach and its outcome; the apply `catchUp: AfterPush` asked for; and retention. + +### Which path owns a request + +The named `GitTarget`'s `KubeProvider` alone picks the path that serves a request, its owner: + +| The request is on | The `GitTarget`'s `KubeProvider` | What happens | +| --- | --- | --- | +| the control cluster | `inCluster` | the control-cluster path serves it | +| the control cluster | a kubeconfig, with a `Reconcile` value | left alone: the kubeconfig may point back at the control cluster, and then its watch serves it | +| the control cluster | a kubeconfig, with `commitRequests: Ignore` | `NotAllowed`, saying to create the request on that cluster | +| a source cluster that provider P watches | P | P's watch serves it | +| a source cluster that provider P watches | any other provider | left alone | +| a source cluster | `gitTargetRef` names no namespace | not picked up, and no status | +| either | the `GitTarget` does not exist | `GitTargetNotFound` once `attachTimeout` has passed since creation | + +Three rules keep several observers of one request safe: + +1. **A path writes only what it knows to be true.** Only the owner serves a request and writes its + progress. A path that is not the owner writes only `GitTargetNotFound`, and only into an empty + status, because nothing else is true for every observer. +2. **Every status write is conditional on the resourceVersion its decision was made from.** On a + conflict the writer reads the request again and decides again, so a `GitTargetNotFound` decided + from an older version never overwrites an owner that has started. +3. **The owner claims a request by writing its first status, and attaches only after that.** The + first write records the author and `Ready=Unknown`, reason `Saving`. If it conflicts and the fresh + object already holds an outcome, the owner stops without saving. After the claim only the owner + writes, so a later conflict comes from its own writes, and an outcome the worker produced is + written onto the fresh object as it is; the save is never restarted. + +Ownership is structural, not "the process that wrote the status": after a restart, the owner is +still the path the `KubeProvider` picks, finds its own claim, and resumes with the author it +recorded. + +A control-cluster request for a `GitTarget` that mirrors another cluster is refused rather than +served, because its author would come from the control cluster's audit and its edits' from the +other cluster's. A request on a source cluster must name the `GitTarget`'s namespace, because that +namespace belongs to the control cluster; the path cannot refuse a request without one, since a +kubeconfig pointing back at the control cluster sees the control cluster's requests, where an +omitted namespace is valid. Known gap: a control-cluster request whose `GitTarget`'s provider has a +`Reconcile` value and points somewhere else gets no status from anyone. + +### Admission + +A grant in the `GitTarget`'s `spec.allowedCommitRequests` sets exactly one of two fields, enforced in +CEL. `consumerNamespaces` admits the request's namespace on the control cluster, by its labels there, +and only on the control-cluster path. `sourceNamespaces` admits it on a source cluster, by name or +`"*"`, only on the source-cluster path, and only when the `KubeProvider` lends that namespace to the +`GitTarget` in `spec.allowedGitTargets`. Omitted, the list is `[{consumerNamespaces: {from: Same}}]`. +A request no grant admits is `NotAllowed`, with a message naming the field that applies. The +`GitTarget` that admitted a request is recorded in `status.gitTargetUID`: a `GitTarget` of that name +with another UID replaced it and must admit the request again, and its refusal is written only once +no worker holds the request. + ## Request and window contract -The request identifies the target in `spec.gitTargetRef.name`, may provide `spec.message`, and sets -`spec.window` (which window to attach to, how long to wait for one, and the timers that close it) and -`spec.whenNothingToCommit`. It is handled by the target’s single branch worker, +The request identifies the target in `spec.gitTargetRef`, may provide `spec.message`, and sets +`spec.window` (which window to attach to, how long to wait for one, and the timers that close it), +`spec.whenNothingToCommit` and `spec.catchUp`. It is handled by the target’s single branch worker, so resource events and the attach request share one FIFO. The worker attaches a request only when all of these match an open window: @@ -21,9 +94,8 @@ The worker attaches a request only when all of these match an open window: 2. the named actor, if either side has one; and 3. whether each side names an actor (`AttributionOutcome.NamesActor`). -The third rule keeps independently configured command admission and live-resource attribution from being -coupled. A request with no named submitter can attach to either a configured-author or an unresolved live -window, but never to a named actor’s window. A request with a named submitter can attach only to that +A request with no named author can attach to either a configured-author or an unresolved live +window, but never to a named actor’s window. A request with a named author can attach only to that actor’s named window. Therefore one user’s request never finalizes another user’s work. A window is only ever opened by a write; a request attaches to one and never opens one. On first @@ -39,9 +111,10 @@ runs from the attach, and so does the first idle interval. A waiting request get the target's own timers are applied, so a target with `idleTimeout: 0s` does not close the window before the request can attach. -`attachTimeout` defaults to `"2s"` rather than `"0s"` because the write a request exists to publish -reaches the worker strictly after the request does: a watch event is held until its audit fact -arrives, so a zero wait is shorter than the smallest window-open latency the pipeline can produce. +`attachTimeout` defaults to `"5s"` rather than `"0s"`, and counts from the worker's registration, +which is after the request's own author was found. The write a request exists to publish reaches +the worker through the same audit join, and on a busy cluster after it, so a zero wait is shorter +than the window-open latency the pipeline can produce. The two durations are separate settings with separate clocks: `attachTimeout` runs from registration, `maxDuration` from the attach. Each is a pointer, so an omitted value and an explicit `"0s"` stay distinguishable. A zero `attachTimeout` attaches to a window already open, or gives up at @@ -75,31 +148,36 @@ pushed commit is required; `Ready=True` also includes successful no-commit outco ## Authorship -The validating admission webhook captures a command submitter before the `CommitRequest` persists. The -controller performs one best-effort, present-or-never lookup: +A request's author comes from its own audit event, on the cluster it was created on, the same way a +watched write's does: the core looks it up in the same fact index, on the audit route of the +`GitTarget`'s `KubeProvider`, keyed by the request's UID and the resourceVersion it was created with, +and waits up to `--author-attribution-grace`. It acquires the fact stream of `commitrequests` on that +route, so the fact follower reads those audit facts at all. + +| Outcome | The request | +|---|---| +| attribution is off | names no author, and nothing waits for one | +| a fact names the creator | names them, in `status.author` | +| no fact, on a provider whose `AuditFactsReceived` is not `True` | names no author: the cluster sends no audit | +| no fact, on a provider that has received facts | `Ready=True`, reason `AuthorUnresolved`: its fact went missing, so it saves nothing rather than take the window that should stay unattributed | -| Admission result | `AuthorAttributed` | Request claim | -|---|---|---| -| submitter record found | `True` / `AttributedFromAdmission` | that named actor | -| capture ran but no record | `False` / `CommitterFallback` | no actor | -| webhook disabled or Redis unavailable | `False` / `AuthorCaptureDisabled` | no actor | +The wait runs whenever attribution is on; whether the provider has received facts is checked only +after it, so a request is not refused while its provider's first audit event is still in transit. +The author is written in the claim: every status write moves the resourceVersion the fact is keyed +by, so a later lookup could never match. -`AuthorAttributed=False` is not a statement about the eventual Git author. The matched live window decides -that: +The author decides which window the request attaches to. The matched window decides the Git author: - configured-author, replay, and resync windows use the configured committer; - a resolved live attribution fact uses the authenticated actor; and - a live attribution miss uses `unknown (attribution unresolved) `. -See [CommitRequest admission authorship](commitrequest-admission-authorship.md) for capture provenance and -[architecture: author identity](../architecture.md#author-and-committer-identity-in-git) for the three Git +See [architecture: author identity](../architecture.md#author-and-committer-identity-in-git) for the three Git author states. ## Lifecycle and outcomes -The controller stamps its conditions on first reconcile, immediately attaches the request, and polls the -worker. There is no audit wait for a `CommitRequest`; a delayed admission record cannot arrive after the -object is visible. +The owner claims the request, attaches it at once, and polls the worker. | Outcome | Conditions | |---|---| @@ -108,18 +186,23 @@ object is visible. | Nothing to commit, with `whenNothingToCommit: CommitEmpty` | `Ready=True`, `Pushed=True`, reason `NoWindow` or `AlreadyPresent`; `status.commit` is the empty commit | | Window produced no diff, and the remote agreed | `Ready=True`, `Pushed=False`, reason `AlreadyPresent` | | Terminal write failure, withdrawal before the worker holds it, shutdown with an unresolved held write, or a target that may not be written | `Ready=False`, `Pushed=False`, `Stalled=True`, reason `FinalizeFailed` | +| Refused by the grants, or made on the wrong cluster | `Ready=False`, `Stalled=True`, reason `NotAllowed` | +| The `GitTarget` did not exist within `attachTimeout` | `Ready=False`, `Stalled=True`, reason `GitTargetNotFound` | +| An earlier save of the same author on the `GitTarget` still in progress | `Ready=False`, `Stalled=True`, reason `SaveInProgress` | +| No audit fact names the creator, on a cluster that sends them | `Ready=True`, `Pushed=False`, reason `AuthorUnresolved` | A retryable materialization or push failure keeps a held request in `WaitingForPush`; it is not a terminal outcome. Publication has to settle the request or the worker must establish that it will no longer execute it before the controller can report failure. -`Reconciling=True` is the normal in-progress state, with the phase the worker reports as its reason: -`Progressing`, `WaitingForWorker`, `WaitingForWindow`, `CollectingWindow`, `WaitingForPush`. Once the +`Ready=Unknown` with `Reconciling=True` is the normal in-progress state, with the phase the worker +reports as its reason: `Saving` (claimed, not yet registered), `WaitingForWorker`, `WaitingForWindow`, +`CollectingWindow`, `WaitingForPush`. Once the request is `CollectingWindow` or `WaitingForPush` the worker holds it, and only the worker ends it. Before that, a request still unresolved `attachTimeout + maxDuration + 120s` after creation is withdrawn from the worker, which cancels it for good, and fails with `FinalizeFailed`; a GitTarget that never started a worker is named in the message. The withdraw rides the worker's queue behind every attach, and is a no-op for a -request the worker holds, so the controller's failure and the worker's answer cannot disagree. +request the worker holds, so the reported failure and the worker's answer cannot disagree. The withdraw goes to the worker that accepted the request's attach, not to whichever worker the GitTarget names now: a worker shared by several GitTargets outlives a deleted one, and a retired worker finishes its @@ -189,6 +272,22 @@ in the words its `GitTarget` uses. While an outage has paused the branch's intak refused at admission and the controller sends it again; past its safety window, it fails with `FinalizeFailed` only after withdrawal confirms that the worker does not hold it. +### Retention + +A finished request (`Ready=True` or `Stalled=True`) is stamped `configbutler.ai/delete-after`, now +plus `--commit-request-ttl`, and deleted once that time has passed; the delete is conditional on the +version that was judged. The annotation is the contract: removing it keeps the request, and a value +already there is never replaced. A request that asked to catch up is deleted only once `Applied` is +settled. On the control cluster this always applies. On a source cluster only +`commitRequests: ReconcileAndDeleteFinished` stamps and deletes; with `Reconcile` nothing is stamped, +and an annotation stamped before a switch back to `Reconcile` stays without effect. + +Retention belongs to the path that owns the request, or that refused it because no path serves its +`GitTarget`. Another path that deletes finished requests, such as a second provider watching the same +namespace, or the control cluster under a kubeconfig that points back at it, leaves the request +alone, whatever `delete-after` it carries. Once the `GitTarget` is gone no path owns the request, +and any path that deletes finished requests may. + ### Capacity, replay, and restart limits The budget charges every kept item its payload bytes, its message, and a fixed overhead for its diff --git a/docs/spec/e2e-serial-registry.md b/docs/spec/e2e-serial-registry.md index c845e343b..c10f5bf62 100644 --- a/docs/spec/e2e-serial-registry.md +++ b/docs/spec/e2e-serial-registry.md @@ -71,11 +71,11 @@ commit it asserts on. The two `audit-consumer`-labelled containers (`Commit Window Batching` ([commit_window_batching_e2e_test.go](../../test/e2e/commit_window_batching_e2e_test.go)), -`Commit Request` ([commit_request_e2e_test.go](../../test/e2e/commit_request_e2e_test.go))) +`Save Request` ([commit_request_e2e_test.go](../../test/e2e/commit_request_e2e_test.go))) were also de-serialized. Their real serial cause was **not** the shared audit *pipeline* — namespace-scoped WatchRules already route each spec's events only to its own GitTarget — but a shared *repo*: both borrowed one Gitea repo from a -`sync.Once` helper and asserted on whole-branch state (`Commit Request` reads +`sync.Once` helper and asserted on whole-branch state (`Save Request` reads `HEAD`/`status.commit`; `Commit Window Batching` reads `git rev-list --count main`), so the *other* audit spec's commits to the same `main` corrupted those reads. Each now provisions its **own** repo via `SetupRepo`, so the only writer to its diff --git a/docs/spec/e2e-test-design.md b/docs/spec/e2e-test-design.md index 0c9b111ff..2e9b4aabb 100644 --- a/docs/spec/e2e-test-design.md +++ b/docs/spec/e2e-test-design.md @@ -251,7 +251,7 @@ The only cluster-level variable still consumed across files is: Repo state (names, checkout paths, Secret names, optional receiver webhook info) flows through a typed `RepoArtifacts` struct returned by `SetupRepo`, not through package-global env vars. -The audit-consumer specs (`Commit Window Batching`, `Commit Request`) each provision their own repo +The audit-consumer specs (`Commit Window Batching`, `Save Request`) each provision their own repo via `SetupRepo`, like every other file. ## Repo Model: One Repo Per E2E Test File diff --git a/docs/spec/status-conditions-guide.md b/docs/spec/status-conditions-guide.md index 938df9c58..c6e35da52 100644 --- a/docs/spec/status-conditions-guide.md +++ b/docs/spec/status-conditions-guide.md @@ -334,34 +334,32 @@ trio; the bounded "did the work actually happen" signal lives on the `Pushed` do ```go const ( - TypeReady = "Ready" // True at a non-error terminal outcome - TypeReconciling = "Reconciling" // True until the request reaches a terminal outcome - TypeStalled = "Stalled" // True when the finalize failed (kstatus Failed) - TypeAuthorAttributed = "AuthorAttributed" // Whether admission captured the command submitter - TypePushed = "Pushed" // True once the commit is in the remote repository + TypeReady = "Ready" // Unknown while the save runs; True at a non-error terminal outcome + TypeReconciling = "Reconciling" // True until the request reaches a terminal outcome + TypeStalled = "Stalled" // True when the request failed (kstatus Failed) + TypePushed = "Pushed" // True once the commit is in the remote repository ) ``` A request's progress runs through the phases the branch worker reports: waiting for a window, -collecting in it, and waiting for the push. `AuthorAttributed` settles at first sight because command -authorship is captured synchronously at admission; it never has an `Unknown` or audit-wait state. +collecting in it, and waiting for the push. Its author is found from its audit event before its first +status write, which records it in `status.author`, so no condition carries it. Canonical reads: -- in progress: `Reconciling=True` reason `Progressing` or `WaitingForWorker`, then `WaitingForWindow`, - `CollectingWindow` or `WaitingForPush` as the worker reports it, `AuthorAttributed` settled, +- in progress: `Ready=Unknown`, `Reconciling=True` reason `Saving` or `WaitingForWorker`, then + `WaitingForWindow`, `CollectingWindow` or `WaitingForPush` as the worker reports it, `Pushed=Unknown` → kstatus InProgress - committed: `Ready=True`, `Pushed=True`, `Stalled=False`, reason `Committed` → kstatus Current -- benign no-commit (nothing to save / already present / foreign open window): `Ready=True`, `Pushed=False`, - `Stalled=False`, with the specific reason on `Ready` (`NoWindow`, `AlreadyPresent`, `WindowMismatch`) - → kstatus Current (a correct, non-error outcome). With `whenNothingToCommit: CommitEmpty`, nothing to - save or already present records an empty commit instead: `Pushed=True`, the reason still naming the - cause -- failed finalize: `Ready=False`, `Stalled=True`, reason `FinalizeFailed` → kstatus Failed - -`AuthorAttributed=True` (`AttributedFromAdmission`) means the command submitter was captured. `False` -(`CommitterFallback`) means capture ran but no admission author record exists; `False` -(`AuthorCaptureDisabled`) means capture is disabled. Both claim no actor and can attach only to an unnamed -window. That condition does not itself determine the Git author: the attached watch window is -configured-author (the committer) when watch attribution is disabled, or explicitly unresolved when watch -attribution ran but could not name an actor. +- benign no-commit (nothing to save / already present / foreign open window / a missing audit fact): + `Ready=True`, `Pushed=False`, `Stalled=False`, with the specific reason on `Ready` (`NoWindow`, + `AlreadyPresent`, `WindowMismatch`, `AuthorUnresolved`) → kstatus Current (a correct, non-error + outcome). With `whenNothingToCommit: CommitEmpty`, nothing to save or already present records an + empty commit instead: `Pushed=True`, the reason still naming the cause +- failed: `Ready=False`, `Stalled=True`, reason `FinalizeFailed`, `NotAllowed`, `GitTargetNotFound` or + `SaveInProgress` → kstatus Failed + +`status.author` empty means no audit fact named the creator, or attribution is off: the request +claims no actor and can attach only to an unnamed window. That does not itself determine the Git +author: the attached watch window is configured-author (the committer) when watch attribution is +disabled, or explicitly unresolved when watch attribution ran but could not name an actor. diff --git a/docs/spec/where-validation-lives.md b/docs/spec/where-validation-lives.md index c003c44ad..b121a68d0 100644 --- a/docs/spec/where-validation-lives.md +++ b/docs/spec/where-validation-lives.md @@ -18,7 +18,7 @@ Validate at the **first** rung that can express the rule: **A validating admission webhook is not a rung on this ladder.** It is only correct when the information being validated *exists only at admission time* and is nowhere on the persisted -object. That is a narrow case, and we ship exactly one instance of it (below). +object. That is a narrow case, and GitOps Reverser ships no webhook at all (below). ## Why cross-object checks go to the reconciler, not a webhook @@ -48,23 +48,16 @@ the same rule: which namespace holds a route depends on every other provider car provider that loses it reports `Stalled=True` (`AuditRouteConflict`) on reconcile rather than being refused at admission. -## The one webhook we ship, and why it is not an exception to the rule +## No webhook ships -[`/validate-operator-types`](../../internal/webhook/validate_operator_types_handler.go#L27) -is **not a validation gate**. It captures the *submitter's identity* on a command kind -(`CommitRequest`) so a commit can be authored by a real Kubernetes user, and for a command it -**always allows**. Identity is the textbook case for admission: it exists in the `AdmissionRequest` -and on no persisted field, so no reconciler could recover it later. See -[`commitrequest-admission-authorship.md`](commitrequest-admission-authorship.md). - -The same endpoint carries one rejection, and it is feedback, not a gate: a `WatchRule` that would -bring a second source namespace to a `GitTarget` declaring `spec.serializeNamespace: false` is -rejected at the moment the mistake is made. The writer refuses the same configuration at the write -whether or not the webhook ran, which is why its `failurePolicy` is `Ignore`. This is the feedback -latency trade above, taken where a webhook is installed anyway, and never as the only gate. - -(`/validate-all`, in [`admission_allow_handler.go`](../../internal/webhook/admission_allow_handler.go#L17), -is an always-allow observation surface wired only by the e2e SUT. The chart does not ship it.) +GitOps Reverser installs no admission webhook. The one it had captured the creator of a +`CommitRequest` at admission, the textbook case for one: identity exists in the `AdmissionRequest` +and on no persisted field. It went once the audit pipeline could give a request's author the same +way it gives an edit's: from the request's own audit event, awaited from the same index with the +same grace ([`attribution.md`](attribution.md)). The same endpoint also rejected a `WatchRule` that +would bring a second source namespace to a `GitTarget` declaring `spec.serializeNamespace: false`. +That was feedback, never the gate: the writer refuses the configuration whether or not the webhook +ran, and the `GitTarget` reports it as `MultipleSourceNamespaces`. ## Applying this diff --git a/hack/crdfields/fields.yaml b/hack/crdfields/fields.yaml index 57d8422be..eb72b9d59 100644 --- a/hack/crdfields/fields.yaml +++ b/hack/crdfields/fields.yaml @@ -128,6 +128,11 @@ kinds: doc: >- Audit route this provider joins. A route is held by one namespace. See [audit route](#audit-route-and-auditfactsreceived) + - path: commitRequests + doc: >- + Whether the `CommitRequest`s created on this cluster are served, and finished ones deleted: + `Ignore`, `Reconcile`, `ReconcileAndDeleteFinished`. See + [serving them](#serving-the-commitrequests-of-a-source-cluster) - kind: GitTarget title: "`GitTarget` (namespaced): which branch and folder" @@ -154,6 +159,11 @@ kinds: - path: commit.message default: the built-in templates doc: How commits are phrased. See [commit messages](commit-messages.md) + - path: allowedCommitRequests + doc: >- + The `CommitRequest`s that may save this target's windows: `consumerNamespaces` on the + cluster GitOps Reverser runs in, `sourceNamespaces` on the one it mirrors. See + [who may save](#who-may-save-specallowedcommitrequests) - path: placement.byType default: the built-in path doc: >- @@ -258,6 +268,15 @@ kinds: doc: >- The `Kustomization` that applies the folders of the `GitTarget`s naming this applier claim. See [who applies this folder](#who-applies-this-folder-specapplierclaimref) + - path: catchUpAfter + doc: >- + How long the `Kustomization` stays behind the branch after a push of GitOps Reverser's own + before an apply of the tip is asked for. Such a push needs no apply for the cluster, but leaves + `status.lastAppliedRevision` and the `GitRepository`'s artifact at the commit before it. The + interval starts when the `Kustomization` is first seen behind, so a burst of saves costs one + apply; the apply can still wait for startup, a retry backoff or edits being published, and an + edit arriving before the resume defers it by an interval. `0s` never asks for this reason, and + drops an apply it asked for that has not started; otherwise `10s` to `24h` - kind: ArgoCDClaim title: "`ArgoCDClaim` (namespaced): the Argo CD `Application` GitOps Reverser asks to sync" @@ -271,12 +290,21 @@ kinds: doc: >- The `Application`, with automated sync off, that syncs the folders of the `GitTarget`s naming this applier claim. See [who applies this folder](#who-applies-this-folder-specapplierclaimref) + - path: catchUpAfter + doc: >- + As a `FluxClaim`'s: how long the `Application`'s last sync stays behind the branch after a push + of GitOps Reverser's own before a sync of the tip is asked for. Off by default, since a sync runs + the `Application`'s hooks and Argo CD's refresh already compares the live state against the tip. + Otherwise `10s` to `24h` - kind: CommitRequest - title: "`CommitRequest` (namespaced): save now" + title: "`CommitRequest` (namespaced, here or on a source cluster): save now" fields: - path: gitTargetRef - doc: The target whose commit window the request attaches to + doc: >- + The target whose commit window the request attaches to: in this namespace, or `namespace` + names another, whose `allowedCommitRequests` must admit this one. On a source cluster + `namespace` is required. See [where a request is created](#where-a-request-is-created) - path: message default: the target's templates doc: Commit message, committed verbatim unless `requestTemplate` frames it @@ -293,3 +321,9 @@ kinds: doc: Close the attached window this long after the attach - path: whenNothingToCommit doc: "`CommitEmpty` records the message in an empty commit when nothing changed" + - path: catchUp + doc: >- + `AfterPush` asks the `GitTarget`'s applier to apply the branch tip right after the commit is + pushed, instead of after the applier claim's `catchUpAfter`. The tip may hold later commits, + and saves pushed close together can share one apply; the `Applied` condition reports it + diff --git a/hack/e2e/deploy-controller.sh b/hack/e2e/deploy-controller.sh index 500b81308..f26b36e65 100755 --- a/hack/e2e/deploy-controller.sh +++ b/hack/e2e/deploy-controller.sh @@ -75,8 +75,8 @@ else kubectl --context "${CTX}" -n "${NAMESPACE}" \ rollout restart "deployment/${deploy}" fi -# 300s: a fresh-namespace install waits on cert-manager issuing the admission -# cert before the pod can mount it, and CI runners near their disk-eviction +# 300s: a fresh-namespace install waits on cert-manager issuing the audit +# certs before the pod can mount them, and CI runners near their disk-eviction # threshold start pods slowly; 180s was too tight in the tail (CI run # 28623232380). On timeout, dump the state that explains *why* the pod never # became ready — node conditions expose DiskPressure, describe/events expose diff --git a/internal/authz/commitrequest_admission.go b/internal/authz/commitrequest_admission.go new file mode 100644 index 000000000..ed14d7f27 --- /dev/null +++ b/internal/authz/commitrequest_admission.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 + +package authz + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// CommitRequestAdmitted reports whether request may save the commit windows of target, the GitTarget +// its spec.gitTargetRef names: a grant in target's spec.allowedCommitRequests admits the request's +// namespace, by the labels that namespace carries. A non-NotFound read of the Namespace is returned as +// err, so the caller retries instead of refusing a request on a transient failure. +func CommitRequestAdmitted( + ctx context.Context, + reader client.Reader, + request *configv1alpha3.CommitRequest, + target *configv1alpha3.GitTarget, +) (Decision, error) { + nsLabels, err := namespaceLabels(ctx, reader, request.Namespace) + if err != nil { + return Decision{}, err + } + admitted, selErr := target.AdmitsCommitRequest(request.Namespace, nsLabels) + if !admitted { + return notAdmitted("GitTarget", client.ObjectKeyFromObject(target), target.DescribeAllowedCommitRequests(), + "spec.allowedCommitRequests", request.Namespace, selErr), nil + } + return Decision{Allowed: true}, nil +} diff --git a/internal/authz/commitrequest_admission_test.go b/internal/authz/commitrequest_admission_test.go new file mode 100644 index 000000000..1cd9cc4fa --- /dev/null +++ b/internal/authz/commitrequest_admission_test.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +package authz + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// A GitTarget admits a CommitRequest by the namespace it is in and that namespace's labels. +func TestCommitRequestAdmitted(t *testing.T) { + tenant := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: testNS, Labels: map[string]string{"platform.example.com/tenant": "true"}, + }} + request := &configv1alpha3.CommitRequest{ObjectMeta: metav1.ObjectMeta{Name: "save", Namespace: testNS}} + target := func(grants ...configv1alpha3.GitTargetCommitRequestGrant) *configv1alpha3.GitTarget { + return &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "shared", Namespace: platformNS}, + Spec: configv1alpha3.GitTargetSpec{AllowedCommitRequests: grants}, + } + } + bySelector := configv1alpha3.GitTargetCommitRequestGrant{ + ConsumerNamespaces: &configv1alpha3.ConsumerNamespaces{ + From: configv1alpha3.GrantFromSelector, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"platform.example.com/tenant": "true"}}, + }, + } + all := configv1alpha3.GitTargetCommitRequestGrant{ + ConsumerNamespaces: &configv1alpha3.ConsumerNamespaces{From: configv1alpha3.GrantFromAll}, + } + reader := fake.NewClientBuilder().WithScheme(admissionScheme(t)).WithObjects(tenant).Build() + + decision, err := CommitRequestAdmitted(context.Background(), reader, request, target()) + require.NoError(t, err) + assert.False(t, decision.Allowed, "the default admits only the GitTarget's own namespace") + assert.Equal(t, ReasonNamespaceNotAuthorized, decision.Reason) + assert.Contains(t, decision.Message, "spec.allowedCommitRequests") + + decision, err = CommitRequestAdmitted(context.Background(), reader, request, target(bySelector)) + require.NoError(t, err) + assert.True(t, decision.Allowed, "the namespace's labels match") + + failing := fake.NewClientBuilder().WithScheme(admissionScheme(t)).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("apiserver unavailable") + }, + }).Build() + _, err = CommitRequestAdmitted(context.Background(), failing, request, target(all)) + require.Error(t, err, "a namespace that cannot be read is retried, never refused") +} diff --git a/internal/controller/access_grants_admission_test.go b/internal/controller/access_grants_admission_test.go index 97cd1bafe..7317fe2dc 100644 --- a/internal/controller/access_grants_admission_test.go +++ b/internal/controller/access_grants_admission_test.go @@ -120,9 +120,46 @@ var _ = Describe("Access grants at admission", func() { refused(kubeProvider("allowedApplierClaims", consumers(same)), "applierNamespaces: Required value") }) - // consumerNamespaces is required on every grant type and from is required inside it, with no default - // for either, so a grant that leaves the "who" out admits nobody instead of its holder's namespace. - // The block's own rules cannot make it required, so each list is checked here. + It("takes a GitTarget's CommitRequest grants with exactly one of consumerNamespaces and sourceNamespaces", func() { + gitTarget := func(grants ...any) *unstructured.Unstructured { + return object("GitTarget", map[string]any{ + "gitBranchRef": map[string]any{"name": "b"}, "path": "apps", + "kubeProviderRef": map[string]any{"name": "k"}, "allowedCommitRequests": grants, + }) + } + from := func(namespaces ...any) map[string]any { return map[string]any{"sourceNamespaces": namespaces} } + accepted(gitTarget(from("shop", "billing"))) + accepted(gitTarget(from("*"))) + accepted(gitTarget(map[string]any{"consumerNamespaces": same}, from("shop"))) + refused(gitTarget(from("{consumer.namespace}")), "variables are refused") + refused(gitTarget(from("Shop")), "a source namespace is a namespace name") + refused(gitTarget(from()), "sourceNamespaces") + refused(gitTarget(map[string]any{}), "exactly one of consumerNamespaces and sourceNamespaces") + refused(gitTarget(map[string]any{"consumerNamespaces": map[string]any{}}), "from: Required value") + both := from("shop") + both["consumerNamespaces"] = same + refused(gitTarget(both), "exactly one of consumerNamespaces and sourceNamespaces") + }) + + It("takes only Ignore for spec.commitRequests beside spec.inCluster", func() { + inCluster := func(commitRequests string) *unstructured.Unstructured { + return object( + "KubeProvider", + map[string]any{"inCluster": map[string]any{}, "commitRequests": commitRequests}, + ) + } + accepted(inCluster("Ignore")) + refused(inCluster("Reconcile"), "spec.commitRequests must be Ignore beside spec.inCluster") + refused(inCluster("ReconcileAndDeleteFinished"), "spec.commitRequests must be Ignore beside spec.inCluster") + accepted(object("KubeProvider", map[string]any{ + "secretRef": map[string]any{"name": "kubeconfig"}, "commitRequests": "ReconcileAndDeleteFinished", + })) + }) + + // consumerNamespaces is required on every grant type but a GitTarget's CommitRequest grant, which + // sets it or sourceNamespaces, and from is required inside it, with no default for either, so a + // grant that leaves the "who" out admits nobody instead of its holder's namespace. The block's own + // rules cannot make it required, so each list is checked here. for _, list := range []struct { name string build func(grant map[string]any) *unstructured.Unstructured diff --git a/internal/controller/commitrequest_apply.go b/internal/controller/commitrequest_apply.go new file mode 100644 index 000000000..d68f950ca --- /dev/null +++ b/internal/controller/commitrequest_apply.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// A CommitRequest with catchUp: AfterPush keeps being reconciled once it is Ready, to +// report in its Applied condition how the apply it asked for went (git.SaveApply). Ready is the +// save's alone and never changes here. + +// crReasonApplyLost is the Applied reason for an apply the worker knows nothing of: GitOps Reverser +// restarted before it learnt how it went, or the record aged out. +const crReasonApplyLost = "Lost" + +// applyLostAfter is how long, from Ready, a request may find its worker knowing nothing of its apply +// before Applied says it is lost. The worker records the apply as the request resolves, so only a +// restart, or a record past its TTL, leaves it unknown. +const applyLostAfter = time.Minute + +// followsApply reports whether a finished request still reports its apply: it asked for one, its +// save ended Ready, and Applied is not settled yet. +func followsApply(cr *configbutleraiv1alpha3.CommitRequest) bool { + return cr.Spec.CatchUp == configbutleraiv1alpha3.CatchUpAfterPush && + conditionIsTrue(cr.Status.Conditions, ConditionTypeReady) && !applyReported(cr) +} + +// applyReported reports whether Applied is settled: True, False, or Unknown because it was lost. +func applyReported(cr *configbutleraiv1alpha3.CommitRequest) bool { + c := apimeta.FindStatusCondition(cr.Status.Conditions, ConditionTypeApplied) + return c != nil && (c.Status != metav1.ConditionUnknown || c.Reason == crReasonApplyLost) +} + +// followApply reads where the request's apply stands, writes it to Applied, and comes back until it +// is settled; then the request expires as any finished one does. A write that conflicts is decided +// again on the next step, from the request as it is stored. +func (c *commitRequestCore) followApply( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, +) (time.Duration, error) { + attach := git.AttachCommitRequest{ + Namespace: cr.Namespace, Name: cr.Name, UID: string(cr.UID), + GitTargetName: cr.GitTargetKey().Name, GitTargetNamespace: cr.GitTargetKey().Namespace, + } + apply, known, err := c.finalizer.LookupCommitRequestApply(ctx, attach) + if err != nil { + p.log.V(1).Info("Cannot read the CommitRequest's apply yet; will retry", + "name", client.ObjectKeyFromObject(cr), "err", err.Error()) + return commitRequestPollInterval, nil + } + if !known { + apply = unknownApply(cr) + } + before := cr.Status.DeepCopy() + setAppliedCondition(cr, apply) + if !equality.Semantic.DeepEqual(*before, cr.Status) { + if err := p.store.updateStatus(ctx, cr); err != nil { + if apierrors.IsConflict(err) { + return commitRequestRequeueNow, nil + } + return 0, client.IgnoreNotFound(err) + } + p.log.Info("CommitRequest apply reported", "name", client.ObjectKeyFromObject(cr), + "reason", apply.Reason, "appliedRevision", cr.Status.AppliedRevision, "message", apply.Message) + } + if applyReported(cr) { + return c.expireFinished(ctx, p, cr) + } + return commitRequestPollInterval, nil +} + +// unknownApply is the apply of a request its worker knows nothing of: still being taken on, shortly +// after Ready, and lost after applyLostAfter. +func unknownApply(cr *configbutleraiv1alpha3.CommitRequest) git.SaveApply { + if ready := apimeta.FindStatusCondition(cr.Status.Conditions, ConditionTypeReady); ready != nil && + time.Since(ready.LastTransitionTime.Time) < applyLostAfter { + return git.SaveApply{Reason: git.SaveApplyApplying, Message: "waiting for the branch worker to take the apply"} + } + return git.SaveApply{Reason: crReasonApplyLost, + Message: "GitOps Reverser does not know how the apply went: it restarted before it learnt, or kept no record"} +} + +// setAppliedCondition writes apply to Applied and status.appliedRevision. +func setAppliedCondition(cr *configbutleraiv1alpha3.CommitRequest, apply git.SaveApply) { + status := metav1.ConditionUnknown + switch { + case apply.Applied: + status = metav1.ConditionTrue + cr.Status.AppliedRevision = apply.AppliedRevision + case apply.Done: + status = metav1.ConditionFalse + } + setCommitRequestCondition(cr, ConditionTypeApplied, status, apply.Reason, apply.Message) +} diff --git a/internal/controller/commitrequest_apply_test.go b/internal/controller/commitrequest_apply_test.go new file mode 100644 index 000000000..22496d666 --- /dev/null +++ b/internal/controller/commitrequest_apply_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +func catchingUp(cr *configv1alpha3.CommitRequest) *configv1alpha3.CommitRequest { + cr.Spec.CatchUp = configv1alpha3.CatchUpAfterPush + return cr +} + +// TestCommitRequestReconcile_ReportsTheApplyItAskedFor: once the save is Ready, a request with +// catchUp: AfterPush is reconciled on, and Applied follows the worker's report until it +// is settled. Ready stays the save's. +func TestCommitRequestReconcile_ReportsTheApplyItAskedFor(t *testing.T) { + c := newCommitRequestClient(t, nil, catchingUp(newCommitRequest("save-1"))) + f := committedFinalizer() + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + + reconcileCommitRequest(t, r, "save-1") + assert.Zero(t, f.applyLookups, "the save resolves first") + + f.apply, f.applyKnown = git.SaveApply{Reason: git.SaveApplyRetrying, Message: "the apply did not complete"}, true + res := reconcileCommitRequest(t, r, "save-1") + assert.Positive(t, res.RequeueAfter, "followed until it is settled") + got := fetchCommitRequest(t, c, "save-1") + requireCondition(t, got, ConditionTypeApplied, metav1.ConditionUnknown, git.SaveApplyRetrying) + requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) + + f.apply = git.SaveApply{ + Done: true, + Applied: true, + Reason: git.SaveApplyApplied, + AppliedRevision: "main@sha1:c0ffee", + } + reconcileCommitRequest(t, r, "save-1") + got = fetchCommitRequest(t, c, "save-1") + requireCondition(t, got, ConditionTypeApplied, metav1.ConditionTrue, git.SaveApplyApplied) + assert.Equal(t, "main@sha1:c0ffee", got.Status.AppliedRevision) + + lookups := f.applyLookups + reconcileCommitRequest(t, r, "save-1") + assert.Equal(t, lookups, f.applyLookups, "settled: not looked up again") +} + +// TestCommitRequestReconcile_ASettledApplyWithoutAnApply: a request that pushed nothing, or whose +// GitTarget names no applier, is Applied=True: no apply was needed, and the cluster already holds what +// it saved. One whose commit the branch lost is Applied=False. None touches Ready. +func TestCommitRequestReconcile_ASettledApplyWithoutAnApply(t *testing.T) { + for _, tc := range []struct { + apply git.SaveApply + want metav1.ConditionStatus + }{ + {git.SaveApply{Done: true, Applied: true, Reason: git.SaveApplyNothingToApply}, metav1.ConditionTrue}, + {git.SaveApply{Done: true, Reason: git.SaveApplyCommitDropped}, metav1.ConditionFalse}, + {git.SaveApply{Done: true, Applied: true, Reason: git.SaveApplyNoApplier}, metav1.ConditionTrue}, + } { + t.Run(tc.apply.Reason, func(t *testing.T) { + c := newCommitRequestClient(t, nil, catchingUp(newCommitRequest("save-1"))) + f := committedFinalizer() + f.apply, f.applyKnown = tc.apply, true + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + + reconcileCommitRequest(t, r, "save-1") + reconcileCommitRequest(t, r, "save-1") + got := fetchCommitRequest(t, c, "save-1") + requireCondition(t, got, ConditionTypeApplied, tc.want, tc.apply.Reason) + requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) + assert.Empty(t, got.Status.AppliedRevision) + }) + } +} + +// TestCommitRequestReconcile_AnApplyNobodyKnowsIsLost: a worker that knows nothing of the apply +// shortly after Ready is still taking it on; long after, it restarted, and Applied says it is lost. +func TestCommitRequestReconcile_AnApplyNobodyKnowsIsLost(t *testing.T) { + c := newCommitRequestClient(t, nil, catchingUp(newCommitRequest("save-1"))) + f := committedFinalizer() + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + reconcileCommitRequest(t, r, "save-1") + + reconcileCommitRequest(t, r, "save-1") + got := fetchCommitRequest(t, c, "save-1") + requireCondition(t, got, ConditionTypeApplied, metav1.ConditionUnknown, git.SaveApplyApplying) + + ready := apimeta.FindStatusCondition(got.Status.Conditions, ConditionTypeReady) + ready.LastTransitionTime = metav1.NewTime(time.Now().Add(-2 * applyLostAfter)) + require.NoError(t, c.Status().Update(context.Background(), &got)) + reconcileCommitRequest(t, r, "save-1") + got = fetchCommitRequest(t, c, "save-1") + requireCondition(t, got, ConditionTypeApplied, metav1.ConditionUnknown, crReasonApplyLost) + assert.False(t, followsApply(&got), "lost is settled") +} + +// TestCommitRequestReconcile_NoCatchUpReportsNoApply: the default asks no apply, and has no Applied. +func TestCommitRequestReconcile_NoCatchUpReportsNoApply(t *testing.T) { + c := newCommitRequestClient(t, nil, newCommitRequest("save-1")) + f := committedFinalizer() + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + + reconcileCommitRequest(t, r, "save-1") + reconcileCommitRequest(t, r, "save-1") + got := fetchCommitRequest(t, c, "save-1") + assert.Nil(t, apimeta.FindStatusCondition(got.Status.Conditions, ConditionTypeApplied)) + assert.Zero(t, f.applyLookups) +} diff --git a/internal/controller/commitrequest_attribution_agreement_test.go b/internal/controller/commitrequest_attribution_agreement_test.go deleted file mode 100644 index 80b44bf5e..000000000 --- a/internal/controller/commitrequest_attribution_agreement_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "testing" - "time" - - "github.com/go-logr/logr" - "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - k8stypes "k8s.io/apimachinery/pkg/types" - - configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" - "github.com/ConfigButler/gitops-reverser/internal/git" - "github.com/ConfigButler/gitops-reverser/internal/queue" - "github.com/ConfigButler/gitops-reverser/internal/watch" -) - -// absentLookup is an attribution index that never has a fact — the production shape of -// "attribution is enabled but nothing matched within the grace". -type absentLookup struct{} - -func (absentLookup) Await( - _ context.Context, - _ queue.FactQuery, - _ time.Duration, -) queue.AuthorResolution { - return queue.AuthorResolution{Result: queue.AttributionAbsent} -} - -// windowOutcomeAttributionOff is the outcome a commit window carries in configured-author mode. -// It is read off a zero git.Event on purpose rather than written as a constant: that IS the -// production value, because watch.Manager.attachAuthor returns early without assigning -// Attribution when the Manager's AuthorResolver is nil. -func windowOutcomeAttributionOff() git.AttributionOutcome { - return git.Event{}.Attribution -} - -// windowOutcomeAttributionMissed drives the real resolver over a lookup that has no fact, which -// is what a live watch event gets when attribution is on and nothing matched in the grace. -func windowOutcomeAttributionMissed(t *testing.T) git.AttributionOutcome { - t.Helper() - _, outcome := watch.NewAuthorResolver(absentLookup{}, 0, logr.Discard(), nil).ResolveAuthor( - context.Background(), - watch.AuthorQuery{ - AuditRoute: "default", - GVR: schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, - UID: k8stypes.UID("uid-1"), - ResourceVersion: "101", - ExactCapable: true, - }, - ) - return outcome -} - -// TestCommitRequest_OutcomesAgree is the cross-subsystem half of the P0 regression test. -// -// The gap that let the blocker through was that every existing test set both sides of the -// comparison to the same literal, so nothing ever checked that the two INDEPENDENT producers -// emit compatible values. This drives each side from its real source — the controller's -// gitOutcome projection, and the watch package's actual resolver / actual zero event — and -// asserts they agree on the only thing matching may depend on across that boundary. -// -// The default deployment is the first row: --admission-webhook defaults to false, so the -// controller emits attributionNotAttempted, while --author-attribution=false leaves the window's -// outcome at the zero value. When those two were different strings, no CommitRequest could -// attach to any window and the user's commit message was silently dropped. -func TestCommitRequest_OutcomesAgree(t *testing.T) { - tests := []struct { - name string - requestSide commitRequestAttribution - windowSide git.AttributionOutcome - wantNamesActor bool - }{ - { - name: "default deployment: webhook off, attribution off", - requestSide: attributionNotAttempted, - windowSide: windowOutcomeAttributionOff(), - }, - { - name: "webhook off, attribution on but missed", - requestSide: attributionNotAttempted, - windowSide: windowOutcomeAttributionMissed(t), - }, - { - name: "webhook on but no record, attribution off", - requestSide: attributionCommitter, - windowSide: windowOutcomeAttributionOff(), - }, - { - name: "webhook on but no record, attribution on but missed", - requestSide: attributionCommitter, - windowSide: windowOutcomeAttributionMissed(t), - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - requestOutcome := tc.requestSide.gitOutcome() - assert.Equal(t, tc.wantNamesActor, requestOutcome.NamesActor()) - assert.Equal(t, requestOutcome.NamesActor(), tc.windowSide.NamesActor(), - "the CommitRequest side produced %q and the window side produced %q; they must "+ - "agree on whether an actor was named, or the request cannot attach and the "+ - "user's commit message is silently dropped", - requestOutcome, tc.windowSide) - }) - } -} - -// TestAttributionOutcome_OffLeavesZeroOutcome states the invariant the default deployment rests on, -// separately from the agreement table so a regression names itself precisely. -func TestAttributionOutcome_OffLeavesZeroOutcome(t *testing.T) { - assert.Equal(t, git.AttributionNotAttempted, windowOutcomeAttributionOff(), - "configured-author mode must leave the event's outcome at AttributionNotAttempted") - assert.Equal(t, git.AttributionNotAttempted, attributionNotAttempted.gitOutcome(), - "a CommitRequest with command-author capture off must carry AttributionNotAttempted") - assert.Equal(t, git.AttributionUnresolved, windowOutcomeAttributionMissed(t), - "attribution that ran and found nothing must be AttributionUnresolved, not the zero value") -} - -// TestCommitRequestAttribution_GitOutcome keeps every controller outcome's projection explicit, -// including the defensive zero value. -func TestCommitRequestAttribution_GitOutcome(t *testing.T) { - tests := []struct { - name string - input commitRequestAttribution - want git.AttributionOutcome - }{ - {name: "unset is not attempted", input: attributionUnset, want: git.AttributionNotAttempted}, - {name: "admission capture is resolved", input: attributionFromAdmission, want: git.AttributionResolved}, - {name: "admission miss is unresolved", input: attributionCommitter, want: git.AttributionUnresolved}, - {name: "capture disabled is not attempted", input: attributionNotAttempted, want: git.AttributionNotAttempted}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, tc.input.gitOutcome()) - }) - } -} - -// TestCommitRequestAttribution_UnsetStatusIsCaptureDisabled keeps an unset decision visible and safe. -func TestCommitRequestAttribution_UnsetStatusIsCaptureDisabled(t *testing.T) { - var request configv1alpha3.CommitRequest - - setCommitRequestAttributed(&request, attributionUnset) - - requireCondition(t, request, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) -} diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index a99ef7321..a7676365a 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -4,11 +4,9 @@ package controller import ( "context" - "errors" + "sync" "time" - "github.com/go-logr/logr" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -19,6 +17,7 @@ import ( configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/queue" + "github.com/ConfigButler/gitops-reverser/internal/watch" ) // CommitRequestFinalizer is the EventRouter seam the reconciler drives, using the @@ -37,20 +36,16 @@ type CommitRequestFinalizer interface { // answer: resolved with git.ErrCommitRequestWithdrawn when it is cancelled, the real outcome // when the worker got there first, or resolved=false with the worker's phase while it decides. WithdrawCommitRequest(ctx context.Context, attach git.AttachCommitRequest) (git.FinalizeResult, bool, error) -} - -// CommandAuthorLookup resolves the author of a CommitRequest from the submitter -// captured at admission by the validate-operator-types webhook, keyed by the persisted -// object's UID. *queue.CommandAuthorStore satisfies it without adaptation. The lookup -// is present-or-never (docs/spec/commitrequest-admission-authorship.md): a miss is -// immediate and final — the webhook is not configured (or a best-effort write missed) — -// and the request claims no actor with no wait. -type CommandAuthorLookup interface { - LookupCommandAuthor(ctx context.Context, uid types.UID) (queue.CommandAuthor, bool) + // LookupCommitRequestApply reports where the apply a request with catchUp: + // AfterPush asked for stands, and false while the worker knows nothing of it. + LookupCommitRequestApply(ctx context.Context, attach git.AttachCommitRequest) (git.SaveApply, bool, error) } // windowMismatchMessage explains the author-bound refusal: an open window // existed but was not this requester's, so it was deliberately left alone. +const saveInProgressMessage = "an earlier save of the same author on this GitTarget was still waiting for " + + "or collecting a commit window, which carries one message; save again once it resolved" + const windowMismatchMessage = "the open commit window belongs to a different author or GitTarget; " + "nothing was committed for this request" @@ -68,14 +63,13 @@ const ( commitRequestPollInterval = 2 * time.Second // commitRequestPushAllowance is what the safety bound allows on top of a request's own - // timers, for the push cooldown and its retries. Authorship is settled synchronously at first - // sight, so no attribution wait belongs here. + // timers: the wait for its author, the push cooldown and its retries. commitRequestPushAllowance = 120 * time.Second // The request window defaults mirror the +kubebuilder:default markers on CommitRequestWindow. // The API server fills them in, so these apply only to a client that bypasses defaulting; // resolving to the same values keeps the two paths from disagreeing. - defaultAttachTimeout = 2 * time.Second + defaultAttachTimeout = 5 * time.Second defaultMaxDuration = 2 * time.Second ) @@ -124,602 +118,139 @@ func (w requestWindow) resolveTimeout() time.Duration { return w.attachTimeout + w.maxDuration + commitRequestPushAllowance } -// CommitRequestReconciler deliberately does NOT use reconcileStatus, which every other controller -// here shares. The difference is not stylistic, so it is worth saying once: -// -// - Its For() carries no GenerationChangedPredicate, so a status write re-enqueues the object. -// The lost-write hazard reconcileStatus exists to close — the winning status-only update being -// filtered out, leaving nothing to correct the stale answer — cannot arise. -// - It reads through APIReader (uncached) precisely so a cache echo cannot re-drive work, which -// is the same lag that makes reconcileStatus lose its optimistic lock in the first place. -// - The policies are opposites, and reconcileStatus's is the wrong one here. It drops a losing -// write and asks the caller to come back and recompute; a re-run of THIS reconcile would -// re-finalize an already-flushed window. So writeTerminalStatus retries in place, bounded, and -// gives up rather than requeue. (The worker now defends the same boundary from its side: a -// request whose window has committed stays identifiable until the push settles it, so a -// re-sent attach is recognized rather than registered afresh. This policy is still the right -// one, but a re-run is no longer the only thing standing between a flushed window and a -// NoOpenWindow that never happened.) -// -// CommitRequestReconciler drives a CommitRequest through its state machine -// (docs/spec/commitrequest-design.md and -// docs/spec/commitrequest-admission-authorship.md): +// progressRecorded reports whether the request's status already says this phase, and why it is +// held, so a poll writes only what changed. An empty phase says nothing new. +func progressRecorded(cr *configbutleraiv1alpha3.CommitRequest, phase git.CommitRequestPhase, held string) bool { + current := findCondition(cr.Status.Conditions, ConditionTypeReconciling) + if current == nil { + return false + } + if phase == "" { + return true + } + reason, message := progressFor(phase, held) + return current.Reason == reason && current.Message == message +} + +// CommitRequestReconciler is the control-cluster path of a CommitRequest: it picks up the requests +// on the cluster GitOps Reverser runs in, with the operator's own client, and hands each step to +// commitRequestCore, which serves both paths. // -// 1. ATTRIBUTE — a single synchronous read of the submitter captured at admission -// (present-or-never). A hit names that submitter as the author -// (AuthorAttributed=True); a miss claims no actor immediately -// (AuthorAttributed=False). There is no wait and no requeue for the -// author: the record is written before the object is visible, so waiting cannot -// help. -// 2. ATTACH + POLL — the instant the author is settled, send the attach to the -// GitTarget's worker, which attaches the request to the author's window with its -// message and timers, and poll the outcome. The attach wait is anchored at the worker's -// registration, so there is no controller-side delay. While it polls, the controller -// reports the phase the worker reports. No window before the attach deadline resolves -// NoWindow; a window belonging to someone else stays open for its own author. +// It reads through APIReader (uncached), so a cache echo of its own status write can never re-drive a +// step that already happened: every step decides from the object as it is stored. Its For() carries +// no GenerationChangedPredicate, so a status or metadata write, such as an edit of the delete-after +// annotation, reconciles the request again. type CommitRequestReconciler struct { client.Client Scheme *runtime.Scheme - // APIReader performs uncached reads so a stale cache echo of our own - // status stamp can never re-run a finalize that already reached a - // terminal phase. Nil falls back to the (cached) Client. + // APIReader performs uncached reads of the requests. Nil falls back to the (cached) Client. APIReader client.Reader - // Finalizer attaches the request to the author-bound open window and reports - // its outcome; AuthorLookup resolves the submitter captured at admission. When - // AuthorLookup is nil (the validate-operator-types webhook is disabled), requests - // claim no actor immediately, with AuthorAttributed=False. The attached window - // determines the eventual Git author. - Finalizer CommitRequestFinalizer - AuthorLookup CommandAuthorLookup + // Finalizer attaches the request to its author's window and reports its outcome. Nil leaves + // every request alone. + Finalizer CommitRequestFinalizer + + // Authors finds a request's creator from audit facts, as a watched write's author is. Nil means + // author attribution is off: a request then names no author, and nothing waits for one. + Authors watch.AuthorResolver + // FactStreams is the set the attribution fact follower reads; the reconciler adds the stream of + // commitrequests on each audit route a creator is looked up on. Nil with attribution off. + FactStreams *queue.FactStreamSet // TTL is how long a request is kept once it finishes. It is written onto the request as // CommitRequestDeleteAfterAnnotation at that moment; zero writes nothing, so the request stays. TTL time.Duration + + streamsOnce sync.Once + streams *commitRequestStreams } -// CommitRequestDeleteAfterAnnotation carries the RFC 3339 time after which the controller deletes a -// finished CommitRequest. The controller writes it once, when the request finishes, and from then -// on the annotation alone decides: remove it to keep the request, or edit it to move the deletion. -// A request that finished before this annotation existed carries none and is never deleted. +// CommitRequestDeleteAfterAnnotation carries the RFC 3339 time after which a finished CommitRequest is +// deleted. It is written once, when the request finishes, and from then on the annotation alone +// decides: remove it to keep the request, or edit it to move the deletion. A request that finished +// before this annotation existed carries none and is never deleted. const CommitRequestDeleteAfterAnnotation = "configbutler.ai/delete-after" // +kubebuilder:rbac:groups=configbutler.ai,resources=commitrequests,verbs=get;list;watch;patch;delete // +kubebuilder:rbac:groups=configbutler.ai,resources=commitrequests/status,verbs=get;update;patch -// Reconcile advances one CommitRequest through attribute → attach + poll → -// terminal status. With MaxConcurrentReconciles=1 concurrent CommitRequests are -// serialized by construction, and the worker keys attaches by request identity so -// re-sends across poll requeues are idempotent. +// Reconcile takes one step of a CommitRequest on the control cluster. func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := logf.FromContext(ctx).WithName("CommitRequestReconciler") - - commitRequest, done, err := r.loadActionableCommitRequest(ctx, req) - if err != nil { - return ctrl.Result{}, err - } - if done { - if commitRequest != nil { - return r.expireFinished(ctx, commitRequest) - } - return ctrl.Result{}, nil - } - - if r.Finalizer == nil { - log.V(1).Info("CommitRequest finalize disabled: no Finalizer configured", - "name", req.NamespacedName) - return ctrl.Result{}, nil - } - - if handled, err := r.refusePrunedGitTargetRef(ctx, commitRequest); handled || err != nil { - return ctrl.Result{}, err - } - - // 1. ATTRIBUTE: settle the request's actor synchronously (present-or-never). - // A hit names the admission submitter; a miss claims no actor. Either way the - // decision is final — there is no wait and no requeue for the author. - author, attribution := r.attributeAuthor(ctx, commitRequest) - - if err := r.stampFirstSightConditions(ctx, commitRequest, attribution); err != nil { - return ctrl.Result{}, err - } - - // The schema rejects an invalid literal message at admission, so a rejection here names an - // object the schema could not have seen: one stored before the rule landed. It is terminal - // rather than retried, because CommitRequest.spec is immutable and no apply can repair it. - if err := git.ValidateLiteralCommitMessage(commitRequest.Spec.Message); err != nil { - // The fallback identity: this fails before ServiceCommitRequest, so nothing has - // confirmed the named GitTarget exists. - r.writeTerminalStatus(ctx, log, commitRequest, git.FinalizeResult{}, err, attribution, - commitRequestTarget{}) - return ctrl.Result{}, nil - } - - // 2. ATTACH + POLL: register the attach idempotently the instant we attribute - // (no controller-side delay — the worker anchors the wait at its registration) - // and poll the outcome. - window := resolveRequestWindow(commitRequest.Spec) - attach := git.AttachCommitRequest{ - Namespace: commitRequest.Namespace, - Name: commitRequest.Name, - UID: string(commitRequest.UID), - Author: author.Author, - Attribution: attribution.gitOutcome(), - GitTargetName: commitRequest.Spec.GitTargetRef.Name, - GitTargetNamespace: commitRequest.Namespace, - Message: commitRequest.Spec.Message, - Attach: window.attach, - AttachTimeout: window.attachTimeout, - IdleTimeout: window.idleTimeout, - MaxDuration: window.maxDuration, - CommitEmpty: commitRequest.Spec.WhenNothingToCommit == configbutleraiv1alpha3.NothingToCommitCommitEmpty, - } - if time.Since(commitRequest.CreationTimestamp.Time) >= window.resolveTimeout() { - // Past the bound the controller only withdraws: an attach sent now would sit behind the - // withdraw on the worker's FIFO and could register the request after it was cancelled. - return r.withdrawCommitRequest(ctx, log, req, commitRequest, attribution, attach) - } - return r.attachCommitRequest(ctx, log, req, commitRequest, attribution, attach) + after, err := r.core().serve(ctx, r.path(ctx), req.NamespacedName) + return ctrl.Result{RequeueAfter: after}, err } -// attachCommitRequest sends the attach and reports the outcome once the worker has one. Until then -// it records the worker's phase and polls; a service error is a not-yet-serviceable worker, so it -// polls on the same terms. -func (r *CommitRequestReconciler) attachCommitRequest( - ctx context.Context, - log logr.Logger, - req ctrl.Request, - commitRequest *configbutleraiv1alpha3.CommitRequest, - attribution commitRequestAttribution, - attach git.AttachCommitRequest, -) (ctrl.Result, error) { - result, resolved, serviceErr := r.Finalizer.ServiceCommitRequest(ctx, attach) - if serviceErr != nil || !resolved { - if serviceErr != nil { - log.V(1).Info("CommitRequest attach not yet serviceable; will retry", - "name", req.NamespacedName, "err", serviceErr.Error()) - } - return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase, result.Held) - } - - if result.Err != nil { - log.Error(result.Err, "CommitRequest finalize failed", - "gitTarget", commitRequest.Spec.GitTargetRef.Name, "name", req.NamespacedName) +// core is the shared core over this reconciler's clients. +func (r *CommitRequestReconciler) core() *commitRequestCore { + r.streamsOnce.Do(func() { r.streams = &commitRequestStreams{set: r.FactStreams} }) + return &commitRequestCore{ + reader: r.Client, finalizer: r.Finalizer, authors: r.Authors, streams: r.streams, ttl: r.TTL, } - r.writeTerminalStatus(ctx, log, commitRequest, result, result.Err, attribution, - resolvedCommitRequestTarget(commitRequest)) - return ctrl.Result{}, nil } -// withdrawCommitRequest handles a request past its safety window. A request the worker holds keeps -// polling with no bound of its own: only the worker can say how it ends. Any other request is -// withdrawn, and fails only on the worker's answer, so a request reported failed can never be -// committed afterwards. A withdraw that cannot be serviced yet polls on the same terms. -func (r *CommitRequestReconciler) withdrawCommitRequest( - ctx context.Context, - log logr.Logger, - req ctrl.Request, - commitRequest *configbutleraiv1alpha3.CommitRequest, - attribution commitRequestAttribution, - attach git.AttachCommitRequest, -) (ctrl.Result, error) { - result, resolved, withdrawErr := r.Finalizer.WithdrawCommitRequest(ctx, attach) - if withdrawErr != nil || !resolved { - if withdrawErr != nil { - log.V(1).Info("CommitRequest withdraw not yet serviceable; will retry", - "name", req.NamespacedName, "err", withdrawErr.Error()) - } - // Not answered yet, or the worker holds the request: keep polling. - return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase, result.Held) - } - - if !errors.Is(result.Err, git.ErrCommitRequestWithdrawn) { - // The worker resolved it on its own before the withdraw: report that outcome, as the - // attach path would have. - r.writeTerminalStatus(ctx, log, commitRequest, result, result.Err, attribution, - resolvedCommitRequestTarget(commitRequest)) - return ctrl.Result{}, nil - } - - // Withdrawn. A worker's answer, or a GitTarget that was read, carries the branch, and only - // then is the request's identity known. - target := commitRequestTarget{} - if result.Branch != "" { - target = resolvedCommitRequestTarget(commitRequest) +// path is the control-cluster path: the operator's own client, and finished requests are deleted. +func (r *CommitRequestReconciler) path(ctx context.Context) commitRequestPath { + reader := r.APIReader + if reader == nil { + reader = r.Client } - - message := resolveTimeoutMessage - switch { - case errors.Is(result.Err, git.ErrBranchWorkerStopped): - message = git.ErrBranchWorkerStopped.Error() - case result.Phase == git.PhaseWaitingForWorker: - message = noWorkerTimeoutMessage + return commitRequestPath{ + store: typedCommitRequestStore{reader: reader, writer: r.Client}, + deletes: true, + log: logf.FromContext(ctx).WithName("CommitRequestReconciler"), } - log.Info("CommitRequest was not acted on within the safety window; withdrawn and failed closed", - "name", req.NamespacedName, "reason", message) - r.writeTerminalStatus(ctx, log, commitRequest, git.FinalizeResult{}, errors.New(message), attribution, target) - return ctrl.Result{}, nil } -// pollCommitRequest reports where the worker says the request stands and polls again. -func (r *CommitRequestReconciler) pollCommitRequest( - ctx context.Context, - commitRequest *configbutleraiv1alpha3.CommitRequest, - attribution commitRequestAttribution, - phase git.CommitRequestPhase, - held string, -) (ctrl.Result, error) { - if err := r.recordWorkerPhase(ctx, commitRequest, attribution, phase, held); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: commitRequestPollInterval}, nil +// typedCommitRequestStore reads and writes CommitRequests through a controller-runtime client. An +// Update carries the object's resourceVersion, so every write it makes is conditional on it. +type typedCommitRequestStore struct { + reader client.Reader + writer client.Client } -// stampFirstSightConditions stamps the still-running conditions the first time a request is seen, -// so the object reports its progress (kstatus InProgress) and AuthorAttributed is settled -// immediately. A disabled controller returns before this, so it never stamps. -func (r *CommitRequestReconciler) stampFirstSightConditions( - ctx context.Context, - commitRequest *configbutleraiv1alpha3.CommitRequest, - attribution commitRequestAttribution, -) error { - if findCondition(commitRequest.Status.Conditions, ConditionTypeReady) != nil { - return nil - } - markCommitRequestProgressing(commitRequest, attribution, "", "") - if err := r.Status().Update(ctx, commitRequest); err != nil { - return err +func (s typedCommitRequestStore) get( + ctx context.Context, key types.NamespacedName, +) (*configbutleraiv1alpha3.CommitRequest, error) { + var cr configbutleraiv1alpha3.CommitRequest + if err := s.reader.Get(ctx, key, &cr); err != nil { + return nil, err } - logf.FromContext(ctx).V(1).Info("Stamped CommitRequest in-progress conditions", - "name", client.ObjectKeyFromObject(commitRequest)) - return nil + return &cr, nil } -// refusePrunedGitTargetRef fails a CommitRequest whose spec names no GitTarget, which can only be -// one thing: an object stored before spec.targetRef was renamed to spec.gitTargetRef, whose old -// value the apiserver stopped serving the moment the new CRDs landed. Admission refuses an empty -// name on every path, so nothing else can produce it. -// -// It is terminal rather than retried, and it has to be, because the object cannot be repaired: -// CommitRequest.spec is wholly immutable, so no apply can put the name back. Retrying would spend -// the controller's attention on an object that will never resolve, and would report a transient -// "get GitTarget" error for a permanent condition. The message names the only fix there is. -func (r *CommitRequestReconciler) refusePrunedGitTargetRef( - ctx context.Context, - commitRequest *configbutleraiv1alpha3.CommitRequest, -) (bool, error) { - if commitRequest.Spec.GitTargetRef.Name != "" { - return false, nil - } - - // This path reaches a terminal state without going through writeTerminalStatus, so it carries - // its own increment. It is rare and migration-only, which is exactly the kind of terminal - // state that must not be missing from the counter. Note the status write below is returned for - // requeue, so a failing write re-decides and increments again: see the instrument's doc - // comment on what one increment means. - recordCommitRequestOutcome(ctx, crOutcomeFailed, commitRequestTarget{}) - failCommitRequest(commitRequest, crReasonGitTargetRefPruned, - "spec.gitTargetRef is empty: this request was created before spec.targetRef was renamed, "+ - "and its value was pruned by the upgrade. A CommitRequest spec is immutable, so this "+ - "one cannot be repaired — delete it and create a new one.") - if err := r.Status().Update(ctx, commitRequest); err != nil { - return true, err - } - r.stampDeleteAfter(ctx, commitRequest) - logf.FromContext(ctx).Info( - "CommitRequest names no GitTarget: created before the gitTargetRef rename", - "name", client.ObjectKeyFromObject(commitRequest)) - return true, nil +func (s typedCommitRequestStore) updateStatus(ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest) error { + return s.writer.Status().Update(ctx, cr) } -// attributeAuthor settles the commit author with a single synchronous lookup of the -// submitter captured at admission (present-or-never). It never waits: a nil -// AuthorLookup (the validate-operator-types webhook is disabled) or a miss both claim no -// actor immediately. The miss case is final — the record is written before the object is -// visible, so there is no asynchronous arrival to wait for. -// -// The lookup result is logged at Info: it is the counterpart to the admission handler's -// "recorded command author" line, so a hit/miss pair makes the whole capture→read path -// legible (the first thing to check when a request unexpectedly claims no actor). -func (r *CommitRequestReconciler) attributeAuthor( - ctx context.Context, - commitRequest *configbutleraiv1alpha3.CommitRequest, -) (queue.CommandAuthor, commitRequestAttribution) { - log := logf.FromContext(ctx).WithName("CommitRequestReconciler") - if r.AuthorLookup == nil { - log.Info("command-author lookup disabled (validate-operator-types webhook off); request claims no actor", - "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) - // Capture is off, so nothing was attempted — distinct from a capture that ran and - // found no record, which is what attributionCommitter now means. - return queue.CommandAuthor{}, attributionNotAttempted - } - if author, ok := r.AuthorLookup.LookupCommandAuthor(ctx, commitRequest.UID); ok { - log.Info("command author resolved from admission record", - "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID, "author", author.Author) - return author, attributionFromAdmission - } - log.Info("no admission command-author record found; request claims no actor", - "name", client.ObjectKeyFromObject(commitRequest), "uid", commitRequest.UID) - return queue.CommandAuthor{}, attributionCommitter -} - -// recordWorkerPhase reports the phase the worker says the request is in. It writes only on a -// change, so polling does not re-write status every interval. An empty phase — the worker has not -// registered the request yet — keeps whatever is shown, so the controller never reports a phase -// the worker has not confirmed. -func (r *CommitRequestReconciler) recordWorkerPhase( - ctx context.Context, - commitRequest *configbutleraiv1alpha3.CommitRequest, - attribution commitRequestAttribution, - phase git.CommitRequestPhase, - held string, +func (s typedCommitRequestStore) annotate( + ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest, key, value string, ) error { - if progressRecorded(commitRequest, phase, held) { - return nil - } - markCommitRequestProgressing(commitRequest, attribution, phase, held) - return r.Status().Update(ctx, commitRequest) -} - -// progressRecorded reports whether the request's status already says this phase, and why it is -// held, so a poll writes only what changed. An empty phase says nothing new. -func progressRecorded(cr *configbutleraiv1alpha3.CommitRequest, phase git.CommitRequestPhase, held string) bool { - current := findCondition(cr.Status.Conditions, ConditionTypeReconciling) - if current == nil { - return false - } - if phase == "" { - return true - } - reason, message := progressFor(phase, held) - return current.Reason == reason && current.Message == message -} - -// loadActionableCommitRequest fetches the CommitRequest and short-circuits -// everything that must not reach the finalize: a deleted object, a terminal -// outcome, and — via an uncached re-read — a stale cache echo of our own -// terminal write from a previous invocation (work past this point must happen -// at most once per CommitRequest). done=true means the finalize has nothing -// further to do; the request is returned with it when it is terminal, so its -// expiry can be decided, and nil when it is gone. -func (r *CommitRequestReconciler) loadActionableCommitRequest( - ctx context.Context, - req ctrl.Request, -) (*configbutleraiv1alpha3.CommitRequest, bool, error) { - var commitRequest configbutleraiv1alpha3.CommitRequest - if err := r.Get(ctx, req.NamespacedName, &commitRequest); err != nil { - return nil, true, client.IgnoreNotFound(err) - } - if commitRequestIsTerminal(&commitRequest) { - return &commitRequest, true, nil - } - - if r.APIReader != nil { - if err := r.APIReader.Get(ctx, req.NamespacedName, &commitRequest); err != nil { - return nil, true, client.IgnoreNotFound(err) - } - if commitRequestIsTerminal(&commitRequest) { - return &commitRequest, true, nil - } - } - - return &commitRequest, false, nil -} - -// expireFinished deletes a finished request once the time in its delete-after annotation has -// passed, and otherwise comes back when it will have. No annotation means keep: the TTL flag only -// decides what is WRITTEN at finish, so changing it never reaches back to a request already stamped, -// and a request whose annotation someone removed stays. -// -// The delete is preconditioned on the resourceVersion that was judged, so an edit to the annotation -// that races it fails the delete with a conflict, and the requeue reads the edit. The edit itself -// needs no extra wiring: this controller has no generation filter, so a metadata change reconciles -// the request again. -func (r *CommitRequestReconciler) expireFinished( - ctx context.Context, - commitRequest *configbutleraiv1alpha3.CommitRequest, -) (ctrl.Result, error) { - value, ok := commitRequest.Annotations[CommitRequestDeleteAfterAnnotation] - if !ok || !commitRequest.DeletionTimestamp.IsZero() { - return ctrl.Result{}, nil - } - log := logf.FromContext(ctx).WithName("CommitRequestReconciler") - deleteAfter, parsed := parseDeleteAfter(value) - if !parsed { - // Kept rather than guessed at: an annotation a person mistyped is not consent to delete. - log.Info("CommitRequest delete-after annotation is not an RFC 3339 time; keeping the request", - "name", client.ObjectKeyFromObject(commitRequest), "value", value) - return ctrl.Result{}, nil - } - - if remaining := time.Until(deleteAfter); remaining > 0 { - return ctrl.Result{RequeueAfter: remaining}, nil + patch := client.MergeFromWithOptions(cr.DeepCopy(), client.MergeFromWithOptimisticLock{}) + if cr.Annotations == nil { + cr.Annotations = map[string]string{} } - - uid, resourceVersion := commitRequest.UID, commitRequest.ResourceVersion - err := r.Delete(ctx, commitRequest, client.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}) - if err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - log.Info("Deleted finished CommitRequest past its delete-after time", - "name", client.ObjectKeyFromObject(commitRequest), "deleteAfter", value) - return ctrl.Result{}, nil + cr.Annotations[key] = value + return s.writer.Patch(ctx, cr, patch) } -// parseDeleteAfter reads the annotation's RFC 3339 time, reporting false for anything else. -func parseDeleteAfter(value string) (time.Time, bool) { - t, err := time.Parse(time.RFC3339, value) - return t, err == nil +func (s typedCommitRequestStore) remove(ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest) error { + uid, resourceVersion := cr.UID, cr.ResourceVersion + return s.writer.Delete(ctx, cr, client.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}) } -// stampDeleteAfter writes the delete-after annotation onto a request that has just finished. It -// runs only on the transition to terminal, never on a later reconcile of a terminal request, which -// is what keeps requests that finished before this existed, and requests someone un-stamped, alone. -// -// A value that is already there wins: a submitter may set its own delete-after at creation, and a -// person may extend it in the moment between the terminal status write and this stamp. So the patch -// carries the resourceVersion it was computed from, and a conflict re-reads the request rather than -// overwriting; if the annotation appeared meanwhile, it is left exactly as it is. -// -// A write that still fails is logged, not requeued: the request is then simply kept, the safe -// direction, and a requeue would re-enter the finalize this controller must run at most once. -func (r *CommitRequestReconciler) stampDeleteAfter( - ctx context.Context, - commitRequest *configbutleraiv1alpha3.CommitRequest, -) { - if r.TTL <= 0 { - return - } - log := logf.FromContext(ctx).WithName("CommitRequestReconciler") - reader := r.APIReader - if reader == nil { - reader = r.Client - } - deleteAfter := time.Now().Add(r.TTL).UTC().Format(time.RFC3339) - - current := commitRequest - for attempt := 1; attempt <= commitRequestStatusUpdateAttempts; attempt++ { - if _, set := current.Annotations[CommitRequestDeleteAfterAnnotation]; set { - return - } - patch := client.MergeFromWithOptions(current.DeepCopy(), client.MergeFromWithOptimisticLock{}) - if current.Annotations == nil { - current.Annotations = map[string]string{} - } - current.Annotations[CommitRequestDeleteAfterAnnotation] = deleteAfter - err := r.Patch(ctx, current, patch) - if err == nil || apierrors.IsNotFound(err) { - return - } - if !apierrors.IsConflict(err) { - log.Error(err, "Failed to stamp delete-after on a finished CommitRequest; it will be kept", - "name", client.ObjectKeyFromObject(commitRequest)) - return - } - var fresh configbutleraiv1alpha3.CommitRequest - if getErr := reader.Get(ctx, client.ObjectKeyFromObject(commitRequest), &fresh); getErr != nil { - if !apierrors.IsNotFound(getErr) { - log.Error(getErr, "Failed to re-read CommitRequest to stamp delete-after; it will be kept", - "name", client.ObjectKeyFromObject(commitRequest)) - } - return - } - if fresh.UID != commitRequest.UID { - return - } - current = &fresh - } - log.Error(nil, "Gave up stamping delete-after after repeated conflicts; the request will be kept", - "name", client.ObjectKeyFromObject(commitRequest)) -} - -// commitRequestStatusUpdateAttempts bounds the terminal-status conflict retry. -const commitRequestStatusUpdateAttempts = 3 - -// writeTerminalStatus records the finalize outcome on the CommitRequest, -// retrying on optimistic-concurrency conflicts. Like the audit-consumer path -// it replaces, a permanently failing write is logged and given up on rather -// than returned for requeue: re-running the reconcile would re-finalize an -// already-flushed window and mis-report the outcome as NoOpenWindow. -func (r *CommitRequestReconciler) writeTerminalStatus( - ctx context.Context, - log logr.Logger, - commitRequest *configbutleraiv1alpha3.CommitRequest, - result git.FinalizeResult, - finalizeErr error, - attribution commitRequestAttribution, - target commitRequestTarget, -) { - expectedUID := commitRequest.UID - reader := r.APIReader - if reader == nil { - reader = r.Client - } - - // Once, here, and deliberately not inside the loop below: result and finalizeErr are inputs - // that do not change across attempts, so this is the point the outcome is settled. Recording - // per attempt would over-count every request that hit a conflict. - recordCommitRequestOutcome(ctx, commitRequestOutcome(result, finalizeErr), target) - - current := commitRequest - for attempt := 1; attempt <= commitRequestStatusUpdateAttempts; attempt++ { - applyFinalizeResultToStatus(current, result, finalizeErr, attribution) - - err := r.Status().Update(ctx, current) - if err == nil { - r.stampDeleteAfter(ctx, current) - finalizeError := "" - if finalizeErr != nil { - finalizeError = finalizeErr.Error() - } - readyReason, readyMessage := commitRequestReadyReason(current) - log.Info("CommitRequest finalized", - "name", client.ObjectKeyFromObject(current), - "ready", commitRequestConditionStatus(current, ConditionTypeReady), - "reason", readyReason, - "message", readyMessage, - "authorAttributed", commitRequestConditionStatus(current, ConditionTypeAuthorAttributed), - "pushed", commitRequestConditionStatus(current, ConditionTypePushed), - "branch", current.Status.Branch, - "commit", current.Status.Commit, - "outcome", result.Outcome, - "finalizeError", finalizeError, - "gitTarget", current.Spec.GitTargetRef.Name, - "age", time.Since(current.CreationTimestamp.Time).String()) - return - } - if !apierrors.IsConflict(err) { - log.Error(err, "Failed to write CommitRequest status") - return - } - - log.V(1).Info("Conflict writing CommitRequest status; retrying", "attempt", attempt) - var fresh configbutleraiv1alpha3.CommitRequest - if getErr := reader.Get(ctx, client.ObjectKeyFromObject(commitRequest), &fresh); getErr != nil { - if apierrors.IsNotFound(getErr) { - log.Info("CommitRequest deleted before status could be written; skipping") - return - } - log.Error(getErr, "Failed to re-read CommitRequest for status update") - return - } - // Never stamp the outcome onto a different incarnation or over a - // terminal outcome another writer got in first. - if fresh.UID != expectedUID { - log.Info("CommitRequest UID changed before status could be written; skipping", - "expectedUID", expectedUID, "objectUID", fresh.UID) - return - } - if commitRequestIsTerminal(&fresh) { - return - } - current = &fresh - } - - log.Error(nil, "Gave up writing CommitRequest status after repeated conflicts") -} - -// SetupWithManager sets up the controller with the Manager. -// MaxConcurrentReconciles is pinned to 1 on purpose: the single worker IS the -// multi-CommitRequest ordering design — concurrent CommitRequests for the same -// GitTarget are serialized exactly as a dedicated finalize-coordinator -// goroutine would serialize them, without the extra moving parts (see -// docs/spec/commitrequest-design.md). -// -// Restart recovery is best-effort by design: the -// message is durable in spec.message, so on restart any non-terminal request is -// re-reconciled — author-resolved from the admission cache when present and -// re-attached — which heals the common cases. The one knowingly-accepted gap is a request whose -// commit was already pushed but whose terminal status was not yet written: the -// in-memory outcome is gone, the re-driven attach finds the work already mirrored, -// and it resolves Rejected/AlreadyPresent. We do not build a durable record to -// close that. +// SetupWithManager sets up the controller with the Manager. Each step decides from the stored object +// and the worker keys attaches by request identity, so steps of different requests may run at once; +// a step can wait out the attribution grace for its request's author, so they must, or one request's +// wait would hold up every other. func (r *CommitRequestReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&configbutleraiv1alpha3.CommitRequest{}). - WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + WithOptions(controller.Options{MaxConcurrentReconciles: commitRequestConcurrency}). Named("commitrequest"). Complete(r) } + +// commitRequestConcurrency is how many CommitRequest steps run at once. +const commitRequestConcurrency = 4 diff --git a/internal/controller/commitrequest_controller_test.go b/internal/controller/commitrequest_controller_test.go index 2a65b0a6a..3100c0173 100644 --- a/internal/controller/commitrequest_controller_test.go +++ b/internal/controller/commitrequest_controller_test.go @@ -17,12 +17,25 @@ import ( var _ = Describe("CommitRequest controller", func() { const namespace = "default" + const targetName = "commitrequest-target" - // The suite registers the reconciler in attributed-author mode (a non-nil AuthorLookup - // that never resolves) with a Finalizer that never resolves, so these specs cover - // the in-progress stamp and the terminal short-circuit only; the full - // attribute → attach → terminal flow and the configured-author path are covered by - // the unit tests in commitrequest_controller_unit_test.go. + // The GitTarget the requests name, mirroring through the suite's in-cluster KubeProvider, so the + // control-cluster path owns them. Its branch does not exist; the Finalizer never asks for one. + BeforeEach(func() { + target := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: targetName, Namespace: namespace}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + KubeProviderRef: testKubeProviderRef(), + GitBranchRef: meta.NamespacedObjectReference{Name: "missing-branch"}, + Path: "commitrequest-folder", + }, + } + Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, target))).To(Succeed()) + }) + + // The suite registers the reconciler with a Finalizer that never resolves, so these specs + // cover the in-progress stamp and the terminal short-circuit only; the full attach → + // terminal flow is covered by the unit tests in commitrequest_controller_unit_test.go. It("stamps a freshly created CommitRequest with in-progress conditions", func() { commitRequest := &configbutleraiv1alpha3.CommitRequest{ ObjectMeta: metav1.ObjectMeta{ @@ -30,8 +43,8 @@ var _ = Describe("CommitRequest controller", func() { Namespace: namespace, }, Spec: configbutleraiv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{ - Name: "team-a-config", + GitTargetRef: meta.NamespacedObjectReference{ + Name: targetName, }, Message: "increase checkout API memory", }, @@ -47,7 +60,7 @@ var _ = Describe("CommitRequest controller", func() { g.Expect(reconciling.Status).To(Equal(metav1.ConditionTrue)) ready := apimeta.FindStatusCondition(fetched.Status.Conditions, ConditionTypeReady) g.Expect(ready).NotTo(BeNil()) - g.Expect(ready.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(ready.Status).To(Equal(metav1.ConditionUnknown)) }, 10*time.Second, 200*time.Millisecond).Should(Succeed()) }) @@ -58,21 +71,21 @@ var _ = Describe("CommitRequest controller", func() { Namespace: namespace, }, Spec: configbutleraiv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{ - Name: "team-a-config", + GitTargetRef: meta.NamespacedObjectReference{ + Name: targetName, }, }, } Expect(k8sClient.Create(ctx, commitRequest)).To(Succeed()) key := client.ObjectKeyFromObject(commitRequest) - // Wait for the initial in-progress stamp. + // Wait for the claim. Eventually(func(g Gomega) { var fetched configbutleraiv1alpha3.CommitRequest g.Expect(k8sClient.Get(ctx, key, &fetched)).To(Succeed()) ready := apimeta.FindStatusCondition(fetched.Status.Conditions, ConditionTypeReady) g.Expect(ready).NotTo(BeNil()) - g.Expect(ready.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(ready.Status).To(Equal(metav1.ConditionUnknown)) }, 10*time.Second, 200*time.Millisecond).Should(Succeed()) // Simulate a finalize having recorded the terminal outcome (Ready=True). @@ -114,7 +127,7 @@ var _ = Describe("CommitRequest message schema", func() { Namespace: namespace, }, Spec: configbutleraiv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{Name: "team-a-config"}, + GitTargetRef: meta.NamespacedObjectReference{Name: "team-a-config"}, Message: message, }, } diff --git a/internal/controller/commitrequest_controller_unit_test.go b/internal/controller/commitrequest_controller_unit_test.go index 5d432bc90..5129b8cb0 100644 --- a/internal/controller/commitrequest_controller_unit_test.go +++ b/internal/controller/commitrequest_controller_unit_test.go @@ -12,6 +12,7 @@ import ( meta "github.com/fluxcd/pkg/apis/meta" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,7 +26,6 @@ import ( configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/git" - "github.com/ConfigButler/gitops-reverser/internal/queue" ) // fakeFinalizer records the reconciler's attach calls and replies with a canned @@ -45,6 +45,19 @@ type fakeFinalizer struct { withdrawErr error withdrawPending bool withdrawCalls []git.AttachCommitRequest + + // The apply a request with catchUp: AfterPush asked for, and the lookups made. + apply git.SaveApply + applyKnown bool + applyErr error + applyLookups int +} + +func (f *fakeFinalizer) LookupCommitRequestApply( + _ context.Context, _ git.AttachCommitRequest, +) (git.SaveApply, bool, error) { + f.applyLookups++ + return f.apply, f.applyKnown, f.applyErr } func (f *fakeFinalizer) WithdrawCommitRequest( @@ -64,26 +77,6 @@ func (f *fakeFinalizer) ServiceCommitRequest( return f.result, f.resolved, f.err } -// fakeAuthorLookup is the command-author source stub: found=false models a miss -// (the validate-operator-types webhook is not configured, or a best-effort write missed), -// which the controller resolves to the committer immediately — present-or-never. -type fakeAuthorLookup struct { - author queue.CommandAuthor - found bool - calls int -} - -func (f *fakeAuthorLookup) LookupCommandAuthor( - _ context.Context, _ types.UID, -) (queue.CommandAuthor, bool) { - f.calls++ - return f.author, f.found -} - -func attributedAlice() *fakeAuthorLookup { - return &fakeAuthorLookup{author: queue.CommandAuthor{Author: "alice"}, found: true} -} - // newCommitRequest builds a request created just now, inside its safety window, as the API server // would stamp it. pastTheBound ages one beyond it. func newCommitRequest(name string) *configv1alpha3.CommitRequest { @@ -95,7 +88,7 @@ func newCommitRequest(name string) *configv1alpha3.CommitRequest { CreationTimestamp: metav1.Now(), }, Spec: configv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{Name: "team-a-config"}, + GitTargetRef: meta.NamespacedObjectReference{Name: "team-a-config"}, Message: "save: " + name, }, } @@ -116,24 +109,29 @@ func withReadyCommitted(cr *configv1alpha3.CommitRequest) *configv1alpha3.Commit return cr } -// withInProgress stamps the still-running conditions a post-restart redelivery -// would already carry (Ready=False, Reconciling=True), which is non-terminal. +// withInProgress stamps the claim a post-restart redelivery would already carry (Ready=Unknown, +// Reconciling=True), which is non-terminal. func withInProgress(cr *configv1alpha3.CommitRequest) *configv1alpha3.CommitRequest { apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ - Type: ConditionTypeReady, Status: metav1.ConditionFalse, + Type: ConditionTypeReady, Status: metav1.ConditionUnknown, Reason: string(git.PhaseWaitingForWindow), Message: "in progress", }) apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ Type: ConditionTypeReconciling, Status: metav1.ConditionTrue, Reason: string(git.PhaseWaitingForWindow), Message: "in progress", }) + // Admitted when it was first seen, by the GitTarget withGitTargets gives it. + key := cr.GitTargetKey() + cr.Status.GitTargetUID = "uid-" + key.Namespace + "-" + key.Name return cr } func newCommitRequestClient(t *testing.T, fns *interceptor.Funcs, objects ...client.Object) client.WithWatch { t.Helper() + objects = withGitTargets(objects) scheme := runtime.NewScheme() require.NoError(t, configv1alpha3.AddToScheme(scheme)) + require.NoError(t, corev1.AddToScheme(scheme)) builder := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(objects...). @@ -144,6 +142,70 @@ func newCommitRequestClient(t *testing.T, fns *interceptor.Funcs, objects ...cli return builder.Build() } +// localProvider is the KubeProvider every fixture GitTarget mirrors through: the operator's own +// identity, so the control-cluster path owns its requests. +const localProvider = "local" + +// withGitTargets gives each GitTarget a UID, as the API server would, and a KubeProvider, the in-cluster +// localProvider in its namespace unless it names one. When objects hold no GitTarget at all, it adds the +// one each CommitRequest names: a request is only attached once a GitTarget admitted it. A test that +// passes GitTargets of its own gets only those. +func withGitTargets(objects []client.Object) []client.Object { + present := map[client.ObjectKey]bool{} + for _, obj := range objects { + if target, ok := obj.(*configv1alpha3.GitTarget); ok { + present[client.ObjectKeyFromObject(target)] = true + if target.UID == "" { + target.UID = types.UID("uid-" + target.Namespace + "-" + target.Name) + } + if target.Spec.KubeProviderRef.Name == "" { + target.Spec.KubeProviderRef.Name = localProvider + } + } + } + if len(present) > 0 { + return withLocalProviders(objects) + } + for _, obj := range objects { + request, ok := obj.(*configv1alpha3.CommitRequest) + if !ok || request.Spec.GitTargetRef.Name == "" || present[request.GitTargetKey()] { + continue + } + key := request.GitTargetKey() + present[key] = true + objects = append(objects, &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, Namespace: key.Namespace, UID: types.UID("uid-" + key.Namespace + "-" + key.Name), + }, + Spec: configv1alpha3.GitTargetSpec{KubeProviderRef: meta.NamespacedObjectReference{Name: localProvider}}, + }) + } + return withLocalProviders(objects) +} + +// withLocalProviders adds the in-cluster localProvider to each namespace a GitTarget names it in, unless +// objects hold a KubeProvider of that name already. +func withLocalProviders(objects []client.Object) []client.Object { + present := map[client.ObjectKey]bool{} + for _, obj := range objects { + if provider, ok := obj.(*configv1alpha3.KubeProvider); ok { + present[client.ObjectKeyFromObject(provider)] = true + } + } + for _, obj := range objects { + target, ok := obj.(*configv1alpha3.GitTarget) + if !ok || target.KubeProviderKey().Name != localProvider || present[target.KubeProviderKey()] { + continue + } + present[target.KubeProviderKey()] = true + objects = append(objects, &configv1alpha3.KubeProvider{ + ObjectMeta: metav1.ObjectMeta{Name: localProvider, Namespace: target.KubeProviderKey().Namespace}, + Spec: configv1alpha3.KubeProviderSpec{InCluster: &configv1alpha3.KubeProviderInCluster{}}, + }) + } + return objects +} + func reconcileCommitRequest(t *testing.T, r *CommitRequestReconciler, name string) ctrl.Result { t.Helper() res, err := r.Reconcile(context.Background(), ctrl.Request{ @@ -186,14 +248,13 @@ func TestCommitRequestReconcile_Committed(t *testing.T) { result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc123", Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-1") got := fetchCommitRequest(t, c, "save-1") requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) requireCondition(t, got, ConditionTypePushed, metav1.ConditionTrue, ReasonSucceeded) - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionTrue, crReasonAttributedFromAdmission) requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionFalse, "") requireCondition(t, got, ConditionTypeStalled, metav1.ConditionFalse, "") assert.Equal(t, "abc123", got.Status.Commit) @@ -201,8 +262,7 @@ func TestCommitRequestReconcile_Committed(t *testing.T) { require.Len(t, f.calls, 1) call := f.calls[0] - assert.Equal(t, "alice", call.Author, - "the attach must carry the author captured at admission") + assert.Empty(t, call.Author, "with attribution off, the request names no author") assert.Equal(t, "team-a-config", call.GitTargetName) assert.Equal(t, "default", call.GitTargetNamespace) assert.Equal(t, "save: save-1", call.Message) @@ -217,7 +277,7 @@ func TestCommitRequestReconcile_NoOpenWindow(t *testing.T) { result: git.FinalizeResult{Outcome: git.FinalizeNoOpenWindow, Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-now") @@ -247,7 +307,7 @@ func TestCommitRequestReconcile_CommitEmptyRecordsTheMessageAndKeepsTheCause(t * result: git.FinalizeResult{Outcome: tc.outcome, Commit: "abc123", Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-empty") @@ -273,7 +333,7 @@ func TestCommitRequestReconcile_WindowMismatchIsExplained(t *testing.T) { result: git.FinalizeResult{Outcome: git.FinalizeWindowMismatch, Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-foreign") @@ -293,7 +353,7 @@ func TestCommitRequestReconcile_AlreadyPresentRejected(t *testing.T) { result: git.FinalizeResult{Outcome: git.FinalizeAlreadyPresent, Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-noop") @@ -311,7 +371,7 @@ func TestCommitRequestReconcile_FinalizeErrorBecomesFailed(t *testing.T) { result: git.FinalizeResult{Err: errors.New("commit failed: unreachable remote")}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-fail") @@ -322,34 +382,6 @@ func TestCommitRequestReconcile_FinalizeErrorBecomesFailed(t *testing.T) { requireCondition(t, got, ConditionTypePushed, metav1.ConditionFalse, "") } -// TestCommitRequestReconcile_LookupMissClaimsNoActor verifies that a lookup miss claims no actor -// immediately — present-or-never, no wait. A freshly created CommitRequest attaches at once with a -// blank author and records AuthorAttributed=False (CommitterFallback). The matching window determines -// the Git author. -func TestCommitRequestReconcile_LookupMissClaimsNoActor(t *testing.T) { - cr := newCommitRequest("save-fresh") - cr.CreationTimestamp = metav1.Now() // fresh: the old path would have requeued here - c := newCommitRequestClient(t, nil, cr) - f := &fakeFinalizer{ - result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Branch: "main"}, - resolved: true, - } - lookup := &fakeAuthorLookup{found: false} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: lookup} - - res := reconcileCommitRequest(t, r, "save-fresh") - - assert.Zero(t, res.RequeueAfter, "a miss is final; the request must not poll for an author") - assert.Equal(t, 1, lookup.calls, "the author is looked up exactly once, synchronously") - require.Len(t, f.calls, 1, "the attach is sent immediately on a miss") - assert.Empty(t, f.calls[0].Author, "a miss attaches with a blank, unnamed actor") - - got := fetchCommitRequest(t, c, "save-fresh") - requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) - requireCondition(t, got, ConditionTypePushed, metav1.ConditionTrue, ReasonSucceeded) - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback) -} - // A request whose window was never defaulted must reach the worker carrying the schema defaults, // not zeros. The API server normally fills the block in, so nil arrives here only from a client // that bypassed defaulting — and a zero there would be the old defect: the request expires before @@ -360,7 +392,7 @@ func TestCommitRequestReconcile_AnOmittedWindowUsesTheDefaults(t *testing.T) { require.Nil(t, cr.Spec.Window, "the fixture must leave the block unset") c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{resolved: false} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-default") @@ -383,7 +415,7 @@ func TestCommitRequestReconcile_ExplicitZerosArePreservedFieldByField(t *testing } c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{resolved: false} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-now") @@ -403,7 +435,7 @@ func TestCommitRequestReconcile_NotResolvedReportsTheWorkersPhase(t *testing.T) cr.Spec.Window = &configv1alpha3.CommitRequestWindow{MaxDuration: &metav1.Duration{Duration: 30 * time.Second}} c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{resolved: false} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} res := reconcileCommitRequest(t, r, "save-linger") @@ -414,8 +446,7 @@ func TestCommitRequestReconcile_NotResolvedReportsTheWorkersPhase(t *testing.T) "the window is passed to the worker, not consumed here") got := fetchCommitRequest(t, c, "save-linger") - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionTrue, crReasonAttributedFromAdmission) - requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionTrue, ReasonProgressing) + requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionTrue, crReasonSaving) assert.False(t, commitRequestIsTerminal(&got), "an unresolved request is not terminal") for _, phase := range []git.CommitRequestPhase{ @@ -425,7 +456,7 @@ func TestCommitRequestReconcile_NotResolvedReportsTheWorkersPhase(t *testing.T) reconcileCommitRequest(t, r, "save-linger") got = fetchCommitRequest(t, c, "save-linger") requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionTrue, string(phase)) - requireCondition(t, got, ConditionTypeReady, metav1.ConditionFalse, string(phase)) + requireCondition(t, got, ConditionTypeReady, metav1.ConditionUnknown, string(phase)) } // A phase the worker has not (yet) reported keeps what is shown rather than guessing. @@ -442,13 +473,13 @@ func TestCommitRequestReconcile_ServiceErrorPolls(t *testing.T) { cr.CreationTimestamp = metav1.Now() c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{err: errors.New("get GitTarget: not found")} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} res := reconcileCommitRequest(t, r, "save-transient") assert.Equal(t, commitRequestPollInterval, res.RequeueAfter) got := fetchCommitRequest(t, c, "save-transient") - requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionTrue, ReasonProgressing) + requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionTrue, crReasonSaving) assert.False(t, commitRequestIsTerminal(&got)) } @@ -462,7 +493,7 @@ func TestCommitRequestReconcile_ResolveTimeoutFailsClosed(t *testing.T) { withdrawResult: git.FinalizeResult{Branch: "main", Err: git.ErrCommitRequestWithdrawn}, withdrawResolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-stuck") @@ -482,7 +513,7 @@ func TestCommitRequestReconcile_AHeldRequestOutlivesTheBound(t *testing.T) { cr := pastTheBound(newCommitRequest("save-held")) c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{result: git.FinalizeResult{Phase: phase}} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} res := reconcileCommitRequest(t, r, "save-held") @@ -517,7 +548,7 @@ func TestCommitRequestReconcile_AnUnansweredWithdrawKeepsPolling(t *testing.T) { withdrawResult: reply, withdrawPending: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} res := reconcileCommitRequest(t, r, "save-race") @@ -537,7 +568,7 @@ func TestCommitRequestReconcile_AWithdrawThatLostTheRaceReportsTheRealOutcome(t withdrawResult: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"}, withdrawResolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-late") @@ -557,7 +588,7 @@ func TestCommitRequestReconcile_NoWorkerIsNamedWhenItFailsClosed(t *testing.T) { }, withdrawResolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-no-worker") @@ -570,12 +601,10 @@ func TestCommitRequestReconcile_TerminalShortCircuits(t *testing.T) { cr := withReadyCommitted(newCommitRequest("save-done")) c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{} - lookup := attributedAlice() - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: lookup} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-done") - assert.Zero(t, lookup.calls) assert.Empty(t, f.calls, "a terminal CommitRequest must never re-attach") } @@ -589,16 +618,15 @@ func TestCommitRequestReconcile_StaleCacheEchoDoesNotRefinalize(t *testing.T) { live := newCommitRequestClient(t, nil, terminal) f := &fakeFinalizer{} - r := &CommitRequestReconciler{Client: cached, APIReader: live, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: cached, APIReader: live, Finalizer: f} reconcileCommitRequest(t, r, "save-echo") assert.Empty(t, f.calls) } -// Webhook-disabled mode (no AuthorLookup) never waits: a freshly created CommitRequest -// attaches immediately with a blank author and records AuthorAttributed=False -// (AuthorCaptureDisabled). The matching window determines the Git author. +// A freshly created CommitRequest attaches immediately with a blank author. The matching window +// determines the Git author. func TestCommitRequestReconcile_ConfiguredAuthorCommitsWithoutWaiting(t *testing.T) { cr := newCommitRequest("save-configured-author") cr.CreationTimestamp = metav1.Now() // fresh: a waiting path would requeue instead of commit @@ -607,7 +635,7 @@ func TestCommitRequestReconcile_ConfiguredAuthorCommitsWithoutWaiting(t *testing result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "c0ffee", Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} // AuthorLookup nil + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} res := reconcileCommitRequest(t, r, "save-configured-author") @@ -615,41 +643,18 @@ func TestCommitRequestReconcile_ConfiguredAuthorCommitsWithoutWaiting(t *testing got := fetchCommitRequest(t, c, "save-configured-author") requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) requireCondition(t, got, ConditionTypePushed, metav1.ConditionTrue, ReasonSucceeded) - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) assert.Equal(t, "c0ffee", got.Status.Commit) require.Len(t, f.calls, 1, "the attach is sent immediately, with no attribution wait") assert.Empty(t, f.calls[0].Author, "webhook-disabled mode attaches with a blank author") } -// Webhook-disabled mode settles AuthorAttributed=False (AuthorCaptureDisabled) immediately -// and never parks the request in a "waiting for the author" state: even while the -// attach is still being polled, the request reports progress. -func TestCommitRequestReconcile_ConfiguredAuthorAttributedImmediately(t *testing.T) { - cr := newCommitRequest("save-committer-poll") - cr.CreationTimestamp = metav1.Now() - c := newCommitRequestClient(t, nil, cr) - f := &fakeFinalizer{resolved: false} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} // AuthorLookup nil - - res := reconcileCommitRequest(t, r, "save-committer-poll") - - assert.Equal(t, commitRequestPollInterval, res.RequeueAfter, "an unresolved attach is polled") - require.Len(t, f.calls, 1) - assert.Empty(t, f.calls[0].Author) - - got := fetchCommitRequest(t, c, "save-committer-poll") - requireCondition(t, got, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonAuthorCaptureDisabled) - requireCondition(t, got, ConditionTypeReconciling, metav1.ConditionTrue, ReasonProgressing) - assert.False(t, commitRequestIsTerminal(&got), "in progress, not terminal") -} - // With no Finalizer wired the controller is inert: it neither attaches, stamps any // condition, nor drives the request to a terminal outcome. (Production always wires // the Finalizer; this is the disabled guard at the top of Reconcile.) func TestCommitRequestReconcile_NilFinalizerIsInert(t *testing.T) { cr := newCommitRequest("save-no-finalizer") c := newCommitRequestClient(t, nil, cr) - r := &CommitRequestReconciler{Client: c, APIReader: c} // Finalizer nil, AuthorLookup nil + r := &CommitRequestReconciler{Client: c, APIReader: c} // Finalizer nil res := reconcileCommitRequest(t, r, "save-no-finalizer") @@ -662,7 +667,7 @@ func TestCommitRequestReconcile_NilFinalizerIsInert(t *testing.T) { func TestCommitRequestReconcile_ObjectDeletedIsBenign(t *testing.T) { c := newCommitRequestClient(t, nil) f := &fakeFinalizer{} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "gone") @@ -697,7 +702,7 @@ func TestCommitRequestReconcile_TerminalWriteRetriesOnConflict(t *testing.T) { result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "ddd111", Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-retry") @@ -720,31 +725,28 @@ func TestApplyFinalizeResultToStatus(t *testing.T) { Branch: "main", }, nil, - attributionFromAdmission, ) requireCondition(t, cr, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) requireCondition(t, cr, ConditionTypePushed, metav1.ConditionTrue, ReasonSucceeded) requireCondition(t, cr, ConditionTypeReconciling, metav1.ConditionFalse, "") requireCondition(t, cr, ConditionTypeStalled, metav1.ConditionFalse, "") - requireCondition(t, cr, ConditionTypeAuthorAttributed, metav1.ConditionTrue, crReasonAttributedFromAdmission) assert.Equal(t, "abc", cr.Status.Commit) assert.Equal(t, "main", cr.Status.Branch) }) - t.Run("no window in grace is a benign ready (no admission author)", func(t *testing.T) { + t.Run("no window in grace is a benign ready", func(t *testing.T) { var cr configv1alpha3.CommitRequest applyFinalizeResultToStatus(&cr, - git.FinalizeResult{Outcome: git.FinalizeNoOpenWindow, Branch: "main"}, nil, attributionCommitter) + git.FinalizeResult{Outcome: git.FinalizeNoOpenWindow, Branch: "main"}, nil) requireCondition(t, cr, ConditionTypeReady, metav1.ConditionTrue, crReasonNoWindow) requireCondition(t, cr, ConditionTypePushed, metav1.ConditionFalse, crReasonNoWindow) - requireCondition(t, cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, crReasonCommitterFallback) assert.Empty(t, cr.Status.Commit) }) t.Run("window mismatch surfaces the reason", func(t *testing.T) { var cr configv1alpha3.CommitRequest applyFinalizeResultToStatus(&cr, - git.FinalizeResult{Outcome: git.FinalizeWindowMismatch, Branch: "main"}, nil, attributionFromAdmission) + git.FinalizeResult{Outcome: git.FinalizeWindowMismatch, Branch: "main"}, nil) ready := requireCondition(t, cr, ConditionTypeReady, metav1.ConditionTrue, crReasonWindowMismatch) assert.Equal(t, windowMismatchMessage, ready.Message) }) @@ -752,7 +754,7 @@ func TestApplyFinalizeResultToStatus(t *testing.T) { t.Run("already present is a benign ready", func(t *testing.T) { var cr configv1alpha3.CommitRequest applyFinalizeResultToStatus(&cr, - git.FinalizeResult{Outcome: git.FinalizeAlreadyPresent, Branch: "main"}, nil, attributionFromAdmission) + git.FinalizeResult{Outcome: git.FinalizeAlreadyPresent, Branch: "main"}, nil) requireCondition(t, cr, ConditionTypeReady, metav1.ConditionTrue, crReasonAlreadyPresent) assert.Empty(t, cr.Status.Commit) }) @@ -761,7 +763,7 @@ func TestApplyFinalizeResultToStatus(t *testing.T) { var cr configv1alpha3.CommitRequest applyFinalizeResultToStatus(&cr, git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc"}, - errors.New("boom"), attributionFromAdmission) + errors.New("boom")) requireCondition(t, cr, ConditionTypeReady, metav1.ConditionFalse, crReasonFinalizeFailed) stalled := requireCondition(t, cr, ConditionTypeStalled, metav1.ConditionTrue, crReasonFinalizeFailed) assert.Equal(t, "boom", stalled.Message) @@ -769,7 +771,7 @@ func TestApplyFinalizeResultToStatus(t *testing.T) { t.Run("unknown outcome stalls", func(t *testing.T) { var cr configv1alpha3.CommitRequest - applyFinalizeResultToStatus(&cr, git.FinalizeResult{}, nil, attributionFromAdmission) + applyFinalizeResultToStatus(&cr, git.FinalizeResult{}, nil) requireCondition(t, cr, ConditionTypeReady, metav1.ConditionFalse, crReasonUnexpectedOutcome) stalled := requireCondition(t, cr, ConditionTypeStalled, metav1.ConditionTrue, crReasonUnexpectedOutcome) assert.Contains(t, stalled.Message, "unexpected finalize outcome") @@ -800,7 +802,7 @@ func TestCommitRequestReconciler_PrunedGitTargetRefIsTerminal(t *testing.T) { UID: "cr-uid", }, Spec: configv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{Name: ""}, + GitTargetRef: meta.NamespacedObjectReference{Name: ""}, }, } @@ -865,3 +867,194 @@ func TestCommitRequestReconciler_LiteralValidation(t *testing.T) { }) } } + +// A CommitRequest may name a GitTarget in another namespace, which saves for it only when a grant in its +// spec.allowedCommitRequests admits the request's namespace. Omitted, the list admits the GitTarget's own +// namespace alone; a refused request fails at once, before anything is attached. +func TestCommitRequestAttach_AGitTargetElsewhereMustAdmitTheRequest(t *testing.T) { + platformTarget := func(grants []configv1alpha3.GitTargetCommitRequestGrant) *configv1alpha3.GitTarget { + return &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "shared", Namespace: "platform"}, + Spec: configv1alpha3.GitTargetSpec{AllowedCommitRequests: grants}, + } + } + fromNames := func(names ...configv1alpha3.GrantNamespace) []configv1alpha3.GitTargetCommitRequestGrant { + return []configv1alpha3.GitTargetCommitRequestGrant{{ + ConsumerNamespaces: &configv1alpha3.ConsumerNamespaces{From: configv1alpha3.GrantFromNames, Names: names}, + }} + } + cases := []struct { + name string + grants []configv1alpha3.GitTargetCommitRequestGrant + admitted bool + }{ + {"the default admits only the GitTarget's own namespace", nil, false}, + {"a grant for another namespace", fromNames("team-b"), false}, + {"a grant for the request's namespace", fromNames("default"), true}, + {"every namespace", []configv1alpha3.GitTargetCommitRequestGrant{{ + ConsumerNamespaces: &configv1alpha3.ConsumerNamespaces{From: configv1alpha3.GrantFromAll}, + }}, true}, + {"an empty list admits nobody", []configv1alpha3.GitTargetCommitRequestGrant{}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cr := newCommitRequest("save") + cr.Spec.GitTargetRef = meta.NamespacedObjectReference{Name: "shared", Namespace: "platform"} + c := newCommitRequestClient(t, nil, cr, platformTarget(tc.grants)) + f := &fakeFinalizer{} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + + reconcileCommitRequest(t, r, "save") + + if tc.admitted { + require.NotEmpty(t, f.calls, "the request must be attached") + assert.Equal(t, "platform", f.calls[0].GitTargetNamespace) + assert.Equal(t, "shared", f.calls[0].GitTargetName) + return + } + assert.Empty(t, f.calls, "a refused request is never attached") + var got configv1alpha3.CommitRequest + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Namespace: "default", Name: "save"}, &got)) + ready := apimeta.FindStatusCondition(got.Status.Conditions, ConditionTypeReady) + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, crReasonNotAllowed, ready.Reason) + assert.Contains(t, ready.Message, "spec.allowedCommitRequests") + assert.True(t, apimeta.IsStatusConditionTrue(got.Status.Conditions, ConditionTypeStalled)) + }) + } +} + +// A request whose GitTarget does not exist is not attached: it waits, writing nothing, since no path +// knows it owns it, and once a GitTarget by that name appears it must admit the request first. +func TestCommitRequestAttach_AGitTargetCreatedLaterMustAdmitTheRequest(t *testing.T) { + cr := newCommitRequest("save") + cr.Spec.GitTargetRef = meta.NamespacedObjectReference{Name: "later", Namespace: "platform"} + c := newCommitRequestClient(t, nil, cr, &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "unrelated", Namespace: "platform"}, + }) + f := &fakeFinalizer{resolved: true, result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "c0ffee"}} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + + res := reconcileCommitRequest(t, r, "save") + assert.Empty(t, f.calls, "nothing is attached before a GitTarget admitted the request") + assert.Positive(t, res.RequeueAfter, "it waits for the GitTarget") + got := fetchCommitRequest(t, c, "save") + assert.Empty(t, got.Status.Conditions, "within attachTimeout nothing is written") + + require.NoError(t, c.Create(context.Background(), &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "later", Namespace: "platform", UID: "uid-later"}, + Spec: configv1alpha3.GitTargetSpec{ + KubeProviderRef: meta.NamespacedObjectReference{Name: localProvider}, + AllowedCommitRequests: []configv1alpha3.GitTargetCommitRequestGrant{}, + }, + })) + reconcileCommitRequest(t, r, "save") + + assert.Empty(t, f.calls, "the GitTarget that appeared admits nobody") + got = fetchCommitRequest(t, c, "save") + ready := apimeta.FindStatusCondition(got.Status.Conditions, ConditionTypeReady) + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, crReasonNotAllowed, ready.Reason) +} + +// The GitTarget that admitted a request is recorded by UID, and a refusal by that same GitTarget +// afterwards, once its grants were lowered, does not fail a request an attach may already have reached. +func TestCommitRequestAttach_ALoweredGrantKeepsARequestItAdmitted(t *testing.T) { + cr := newCommitRequest("save") + target := &configv1alpha3.GitTarget{ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "default"}} + c := newCommitRequestClient(t, nil, cr, target) + f := &fakeFinalizer{} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + + reconcileCommitRequest(t, r, "save") + require.Len(t, f.calls, 1) + assert.Equal(t, string(target.UID), f.calls[0].GitTargetUID, "the attach is bound to the admitting GitTarget") + got := fetchCommitRequest(t, c, "save") + assert.Equal(t, string(target.UID), got.Status.GitTargetUID) + + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(target), target)) + target.Spec.AllowedCommitRequests = []configv1alpha3.GitTargetCommitRequestGrant{} + require.NoError(t, c.Update(context.Background(), target)) + reconcileCommitRequest(t, r, "save") + + assert.Len(t, f.calls, 2, "the attach is sent again, as before the grant was lowered") + got = fetchCommitRequest(t, c, "save") + ready := apimeta.FindStatusCondition(got.Status.Conditions, ConditionTypeReady) + require.NotNil(t, ready) + assert.NotEqual(t, crReasonNotAllowed, ready.Reason) +} + +// A GitTarget recreated with grants that refuse a request does not fail it while the worker the +// GitTarget it replaced sent it to may hold it: that worker answers first. A save it committed is +// reported as committed; one nothing held is refused; one it holds is waited for. +func TestCommitRequestAttach_ARefusingReplacementAsksTheWorkerFirst(t *testing.T) { + tests := []struct { + name string + withdraw git.FinalizeResult + resolved bool + wantReason string + wantCommit string + }{ + { + name: "the worker committed it", + withdraw: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "c0ffee", Branch: "main"}, + resolved: true, + wantReason: crReasonCommitted, + wantCommit: "c0ffee", + }, + { + name: "nothing held it", + withdraw: git.FinalizeResult{Err: git.ErrCommitRequestWithdrawn}, + resolved: true, + wantReason: crReasonNotAllowed, + }, + { + name: "the worker holds it", + withdraw: git.FinalizeResult{Phase: git.PhaseWaitingForPush}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cr := newCommitRequest("save") + target := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "default", UID: "uid-first"}, + } + c := newCommitRequestClient(t, nil, cr, target) + f := &fakeFinalizer{} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} + + reconcileCommitRequest(t, r, "save") + require.Len(t, f.calls, 1) + + require.NoError(t, c.Delete(context.Background(), target)) + refusing := configv1alpha3.GitTargetSpec{ + AllowedCommitRequests: []configv1alpha3.GitTargetCommitRequestGrant{}, + } + refusing.KubeProviderRef = meta.NamespacedObjectReference{Name: localProvider} + require.NoError(t, c.Create(context.Background(), &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "default", UID: "uid-second"}, + Spec: refusing, + })) + f.withdrawResult, f.withdrawResolved, f.withdrawPending = tc.withdraw, tc.resolved, true + res := reconcileCommitRequest(t, r, "save") + + assert.Len(t, f.calls, 1, "nothing is attached through the replacement") + require.Len(t, f.withdrawCalls, 1) + assert.Equal(t, "uid-first", f.withdrawCalls[0].GitTargetUID, "the worker is asked about the first one") + got := fetchCommitRequest(t, c, "save") + assert.Equal(t, "uid-first", got.Status.GitTargetUID) + ready := apimeta.FindStatusCondition(got.Status.Conditions, ConditionTypeReady) + require.NotNil(t, ready) + if tc.wantReason == "" { + assert.Positive(t, res.RequeueAfter, "a held request is waited for") + assert.NotEqual(t, metav1.ConditionTrue, ready.Status) + assert.NotEqual(t, crReasonNotAllowed, ready.Reason) + return + } + assert.Equal(t, tc.wantReason, ready.Reason) + assert.Equal(t, tc.wantCommit, got.Status.Commit) + }) + } +} diff --git a/internal/controller/commitrequest_core.go b/internal/controller/commitrequest_core.go new file mode 100644 index 000000000..b95053834 --- /dev/null +++ b/internal/controller/commitrequest_core.go @@ -0,0 +1,583 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/authz" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/queue" + "github.com/ConfigButler/gitops-reverser/internal/watch" +) + +// A CommitRequest is picked up in one of two ways (docs/spec/commitrequest-design.md): by the +// CommitRequestReconciler on the control cluster, or by the CommitRequestServer's watch of a source +// cluster through its KubeProvider. Both hand it to commitRequestCore, which decides everything else +// once, so the two paths cannot drift: who owns the request, whether its GitTarget admits it, who +// created it, the attach and its outcome, the apply it asked for, and how long it is kept. +// +// Three rules keep several observers of one request safe: +// +// 1. Only the owner, the path the named GitTarget's KubeProvider picks, serves the request. A path +// that is not the owner writes only GitTargetNotFound, after attachTimeout, into an empty status, +// because nothing else is true for every observer. +// 2. Every status write is conditional on the resourceVersion its decision was made from. A conflict +// reads the request again and decides again. +// 3. The owner claims a request by writing its first status, the author and Ready=Unknown, before it +// attaches. After the claim only the owner writes, so a later conflict comes from its own writes, +// and an outcome the worker produced is written onto the fresh object as it is. +// +// Every step is decided again from the object as it is stored, so the owner is structural rather than +// "the process that wrote the status": after a restart, the owner finds its own claim and resumes. + +// commitRequestGVR is the served resource of a CommitRequest, on whichever cluster it lives. +func commitRequestGVR() schema.GroupVersionResource { + return configbutleraiv1alpha3.GroupVersion.WithResource("commitrequests") +} + +const ( + // commitRequestStatusAttempts bounds a write's re-read on conflict before the step is decided again. + commitRequestStatusAttempts = 3 + // commitRequestUnownedRecheck is how soon a request left to another path is looked at again: its + // GitTarget's KubeProvider may start or stop serving it. + commitRequestUnownedRecheck = 5 * time.Minute +) + +// commitRequestStore is how one path reads and writes the CommitRequests it serves. +type commitRequestStore interface { + // get reads the request as it is stored. + get(ctx context.Context, key types.NamespacedName) (*configbutleraiv1alpha3.CommitRequest, error) + // updateStatus writes cr's status, conditional on cr.ResourceVersion. On success cr carries the + // version it wrote. + updateStatus(ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest) error + // annotate sets one annotation, conditional on cr.ResourceVersion. + annotate(ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest, key, value string) error + // remove deletes the request, conditional on its UID and resourceVersion. + remove(ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest) error +} + +// commitRequestPath is one way of picking a CommitRequest up. +type commitRequestPath struct { + // provider is the KubeProvider whose watch of a source cluster this is; empty on the control + // cluster. + provider types.NamespacedName + store commitRequestStore + // deletes says whether this path stamps configbutler.ai/delete-after on a request that finishes, + // and deletes a finished request once that time has passed. + deletes bool + log logr.Logger +} + +// controlCluster reports whether this is the control-cluster path. +func (p commitRequestPath) controlCluster() bool { return p.provider.Name == "" } + +// gitTargetKey is the GitTarget a request names. An omitted namespace is the request's own on the +// control cluster; a request on a source cluster must name it, so without one it is not picked up. +func (p commitRequestPath) gitTargetKey(cr *configbutleraiv1alpha3.CommitRequest) (types.NamespacedName, bool) { + if !p.controlCluster() && cr.Spec.GitTargetRef.Namespace == "" { + return types.NamespacedName{}, false + } + return cr.GitTargetKey(), true +} + +// commitRequestCore serves a CommitRequest for either path, one step at a time. +type commitRequestCore struct { + // reader reads GitTargets, KubeProviders and Namespaces on the control cluster. + reader client.Reader + finalizer CommitRequestFinalizer + // authors finds a request's creator from audit facts; nil when author attribution is off. + authors watch.AuthorResolver + // streams holds the fact stream of commitrequests on each audit route a creator is looked up on. + streams *commitRequestStreams + // ttl is how long a finished request is kept, written as its delete-after time; zero writes none. + ttl time.Duration +} + +// serve takes one step for the request key on path p, and says how soon to take the next. Zero means +// nothing is due until the request changes. +func (c *commitRequestCore) serve( + ctx context.Context, p commitRequestPath, key types.NamespacedName, +) (time.Duration, error) { + cr, err := p.store.get(ctx, key) + if err != nil { + return 0, client.IgnoreNotFound(err) + } + if !cr.DeletionTimestamp.IsZero() { + return 0, nil + } + if commitRequestIsTerminal(cr) { + return c.serveFinished(ctx, p, cr) + } + if c.finalizer == nil { + p.log.V(1).Info("CommitRequest finalize disabled: no Finalizer configured", "name", key) + return 0, nil + } + return c.serveOpen(ctx, p, cr) +} + +// serveOpen is the step of a request whose save has not ended: it waits for its GitTarget, leaves it +// to the path that owns it, or claims and drives it as its owner. +func (c *commitRequestCore) serveOpen( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, +) (time.Duration, error) { + if cr.Spec.GitTargetRef.Name == "" { + return 0, c.refusePrunedGitTargetRef(ctx, p, cr) + } + targetKey, named := p.gitTargetKey(cr) + if !named { + return 0, nil + } + target, found, err := c.gitTarget(ctx, targetKey) + if err != nil { + return 0, err + } + if !found { + if commitRequestClaimed(cr) { + // Its GitTarget went away after the claim. Only a worker that holds the request can still + // say how it ends, and that answer is true for every observer. + return c.drive(ctx, p, cr, nil, nil) + } + return c.awaitGitTarget(ctx, p, cr, targetKey) + } + provider, owner, refusal, err := c.owns(ctx, p, target) + if err != nil { + return 0, err + } + if !owner { + if refusal != "" && len(cr.Status.Conditions) == 0 { + return c.decide(ctx, p, cr, commitRequestOutcome{err: fmt.Errorf("%w: %s", errCommitRequestNotAllowed, + refusal)}, false) + } + return commitRequestUnownedRecheck, nil + } + if commitRequestClaimed(cr) { + return c.drive(ctx, p, cr, target, provider) + } + return c.claim(ctx, p, cr, target, provider) +} + +// gitTarget reads the GitTarget key names; found is false when there is none, or it is being deleted. +func (c *commitRequestCore) gitTarget( + ctx context.Context, key types.NamespacedName, +) (*configbutleraiv1alpha3.GitTarget, bool, error) { + var target configbutleraiv1alpha3.GitTarget + if err := c.reader.Get(ctx, key, &target); err != nil { + if apierrors.IsNotFound(err) { + return nil, false, nil + } + return nil, false, fmt.Errorf("read GitTarget %s: %w", key, err) + } + return &target, target.DeletionTimestamp.IsZero(), nil +} + +// owns decides whether path p serves requests for target: the target's KubeProvider alone picks the +// path. On the control cluster that is a provider with inCluster. A provider with a kubeconfig may point +// back at the control cluster, and then its own watch serves the request, so it is left alone, unless +// that provider serves no CommitRequests: then refusal says why, and where to create the request +// instead. A provider that cannot be found picks no path yet. +func (c *commitRequestCore) owns( + ctx context.Context, p commitRequestPath, target *configbutleraiv1alpha3.GitTarget, +) (*configbutleraiv1alpha3.KubeProvider, bool, string, error) { + key := target.KubeProviderKey() + var provider configbutleraiv1alpha3.KubeProvider + if err := c.reader.Get(ctx, key, &provider); err != nil { + if apierrors.IsNotFound(err) { + return nil, false, "", nil + } + return nil, false, "", fmt.Errorf("read KubeProvider %s: %w", key, err) + } + if !p.controlCluster() { + return &provider, key == p.provider, "", nil + } + if provider.IsInCluster() { + return &provider, true, "", nil + } + if provider.ServesCommitRequests() { + return &provider, false, "", nil + } + return &provider, false, fmt.Sprintf("GitTarget %s mirrors another cluster, through KubeProvider %s; create "+ + "the CommitRequest there, and set spec.commitRequests: Reconcile on KubeProvider %s", + client.ObjectKeyFromObject(target), key, key), nil +} + +// awaitGitTarget waits attachTimeout from the request's creation for the GitTarget it names, which may +// be created alongside it, and then reports GitTargetNotFound. That outcome is true for every observer, +// so any path may write it, but only into an empty status: a path that claimed the request in the +// meantime has started, and is never overwritten. +func (c *commitRequestCore) awaitGitTarget( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, key types.NamespacedName, +) (time.Duration, error) { + if len(cr.Status.Conditions) > 0 { + return commitRequestPollInterval, nil + } + wait := resolveRequestWindow(cr.Spec).attachTimeout - time.Since(cr.CreationTimestamp.Time) + if wait > 0 { + return min(wait, commitRequestPollInterval), nil + } + return c.decide(ctx, p, cr, commitRequestOutcome{err: fmt.Errorf("%w: GitTarget %s does not exist", + errGitTargetNotFound, key), fallbackTarget: true}, false) +} + +// admit decides whether target admits the request in spec.allowedCommitRequests: admitted, or the +// refusal, which wraps errCommitRequestNotAllowed. err is a read to retry. +// +// On the control cluster, a grant's consumerNamespaces admit the request's namespace, by its labels +// there. On a source cluster, a grant's sourceNamespaces name the request's namespace, and the +// KubeProvider must lend that namespace to the GitTarget in spec.allowedGitTargets, the bound its +// WatchRules have too. +func (c *commitRequestCore) admit( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, + target *configbutleraiv1alpha3.GitTarget, provider *configbutleraiv1alpha3.KubeProvider, +) (bool, error, error) { + key := client.ObjectKeyFromObject(target) + if p.controlCluster() { + decision, err := authz.CommitRequestAdmitted(ctx, c.reader, cr, target) + if err != nil { + return false, nil, err + } + if !decision.Allowed { + return false, fmt.Errorf("%w: %s; on the cluster GitOps Reverser runs in, a grant admits by "+ + "consumerNamespaces", errCommitRequestNotAllowed, decision.Message), nil + } + return true, nil, nil + } + if !target.AdmitsSourceCommitRequest(cr.Namespace) { + return false, fmt.Errorf("%w: GitTarget %s does not admit CommitRequests from namespace %q of the "+ + "cluster it mirrors; on that cluster, a grant in spec.allowedCommitRequests admits by sourceNamespaces", + errCommitRequestNotAllowed, key, cr.Namespace), nil + } + labels, err := namespaceLabelsOf(ctx, c.reader, target.Namespace) + if err != nil { + return false, nil, err + } + scope, err := provider.AdmitsGitTarget(key, labels) + if err != nil { + return false, nil, err + } + if !scope.AllowsSourceNamespace(cr.Namespace) { + return false, fmt.Errorf("%w: KubeProvider %s does not lend namespace %q to GitTarget %s in "+ + "spec.allowedGitTargets", errCommitRequestNotAllowed, provider.Key(), cr.Namespace, key), nil + } + return true, nil, nil +} + +// claim takes an unclaimed request that p owns: it checks the message and the GitTarget's grants, +// finds the request's creator, and writes the first status, which records the author. Only once that +// write landed is the request attached. +func (c *commitRequestCore) claim( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, + target *configbutleraiv1alpha3.GitTarget, provider *configbutleraiv1alpha3.KubeProvider, +) (time.Duration, error) { + // The schema rejects an invalid literal message at admission, so a rejection here names an object + // the schema could not have seen: one stored before the rule landed. It is terminal rather than + // retried, because the spec is immutable and no apply can repair it. + if err := git.ValidateLiteralCommitMessage(cr.Spec.Message); err != nil { + return c.decide(ctx, p, cr, commitRequestOutcome{err: err, fallbackTarget: true}, false) + } + admitted, refusal, err := c.admit(ctx, p, cr, target, provider) + if err != nil { + return 0, err + } + if !admitted { + return c.decide(ctx, p, cr, commitRequestOutcome{err: refusal}, false) + } + author, refused, err := c.creator(ctx, cr, provider) + if err != nil { + return 0, err + } + if refused { + return c.decide(ctx, p, cr, commitRequestOutcome{authorUnresolved: true}, false) + } + + claimed, err := c.writeClaim(ctx, p, cr, author, string(target.UID)) + if err != nil || !claimed { + return commitRequestRequeueNow, err + } + p.log.Info("CommitRequest claimed", "name", client.ObjectKeyFromObject(cr), + "gitTarget", client.ObjectKeyFromObject(target), "author", author) + return c.drive(ctx, p, cr, target, provider) +} + +// creator is the request's author, found from its audit fact on the route of the GitTarget's +// KubeProvider, as a watched write's author is, and within the same grace. The wait runs whenever +// attribution is on. refused reports a request no fact names on a provider that has received audit +// facts: its fact went missing, and it must not take the window that stays unattributed. +// +// The lookup matches the request's resourceVersion, which every status write moves, so it is made +// only before the claim, and the claim records the answer. +// +// Whether the provider has received audit facts is read after the wait, not from provider: the first +// facts may arrive during it, and a request whose own fact is still missing then must be refused +// rather than take the unnamed window. A provider that cannot be read then is an error, retried. +func (c *commitRequestCore) creator( + ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest, provider *configbutleraiv1alpha3.KubeProvider, +) (string, bool, error) { + if c.authors == nil { + return "", false, nil + } + c.streams.ensure(provider.AuditRoute()) + user, outcome := c.authors.ResolveAuthor(ctx, watch.AuthorQuery{ + AuditRoute: provider.AuditRoute(), + GVR: commitRequestGVR(), + UID: cr.UID, + ResourceVersion: cr.ResourceVersion, + Namespace: cr.Namespace, + Labels: cr.Labels, + Name: cr.Name, + ExactCapable: true, + }) + if outcome.NamesActor() { + return user.Username, false, nil + } + if outcome == git.AttributionNotAttempted { + return "", false, nil + } + var current configbutleraiv1alpha3.KubeProvider + if err := c.reader.Get(ctx, provider.Key(), ¤t); err != nil { + return "", false, fmt.Errorf("read KubeProvider %s after the author wait: %w", provider.Key(), err) + } + return "", apimeta.IsStatusConditionTrue(current.Status.Conditions, KubeProviderConditionAuditFactsReceived), nil +} + +// writeClaim writes the first status of a request: its author, the GitTarget that admitted it, and +// Ready=Unknown. A conflict reads the request again, and the claim is written onto it only while it is +// still unclaimed; otherwise claimed=false, and the next step decides from what is there. +func (c *commitRequestCore) writeClaim( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, author, targetUID string, +) (bool, error) { + for attempt := 1; ; attempt++ { + cr.Status.Author = author + cr.Status.GitTargetUID = targetUID + markCommitRequestProgressing(cr, "", "") + err := p.store.updateStatus(ctx, cr) + if err == nil { + return true, nil + } + if !apierrors.IsConflict(err) || attempt == commitRequestStatusAttempts { + return false, err + } + fresh, same, getErr := c.reread(ctx, p, cr) + if getErr != nil || !same || len(fresh.Status.Conditions) > 0 { + return false, getErr + } + *cr = *fresh + } +} + +// reread reads cr again; same is false when it is gone or was replaced by another request of its name. +func (c *commitRequestCore) reread( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, +) (*configbutleraiv1alpha3.CommitRequest, bool, error) { + fresh, err := p.store.get(ctx, client.ObjectKeyFromObject(cr)) + if err != nil { + return nil, false, client.IgnoreNotFound(err) + } + return fresh, fresh.UID == cr.UID, nil +} + +// attachFor is the attach of a claimed request to its GitTarget's window. +func (c *commitRequestCore) attachFor(cr *configbutleraiv1alpha3.CommitRequest) git.AttachCommitRequest { + window := resolveRequestWindow(cr.Spec) + attribution := git.AttributionNotAttempted + switch { + case cr.Status.Author != "": + attribution = git.AttributionResolved + case c.authors != nil: + attribution = git.AttributionUnresolved + } + key := cr.GitTargetKey() + return git.AttachCommitRequest{ + Namespace: cr.Namespace, + Name: cr.Name, + UID: string(cr.UID), + Author: cr.Status.Author, + Attribution: attribution, + GitTargetName: key.Name, + GitTargetNamespace: key.Namespace, + GitTargetUID: cr.Status.GitTargetUID, + Message: cr.Spec.Message, + Attach: window.attach, + AttachTimeout: window.attachTimeout, + IdleTimeout: window.idleTimeout, + MaxDuration: window.maxDuration, + CommitEmpty: cr.Spec.WhenNothingToCommit == configbutleraiv1alpha3.NothingToCommitCommitEmpty, + CatchUpNow: cr.Spec.CatchUp == configbutleraiv1alpha3.CatchUpAfterPush, + } +} + +// drive advances a claimed request: it sends the attach, or the withdraw once the request is past its +// bound, and records the worker's phase until the worker has an outcome. target and its provider are +// nil when the GitTarget is gone. +// +// A GitTarget with another UID than the one that admitted the request replaced it, and admitted +// nothing: it is asked again. Its admission is recorded before anything is attached to it; its +// refusal is written only once no worker holds the request, because the GitTarget it replaced may +// have had the request committed. +func (c *commitRequestCore) drive( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, + target *configbutleraiv1alpha3.GitTarget, provider *configbutleraiv1alpha3.KubeProvider, +) (time.Duration, error) { + attach := c.attachFor(cr) + if target != nil && string(target.UID) != cr.Status.GitTargetUID { + admitted, refusal, err := c.admit(ctx, p, cr, target, provider) + if err != nil { + return 0, err + } + if !admitted { + return c.refuseOnceUnheld(ctx, p, cr, attach, refusal) + } + cr.Status.GitTargetUID = string(target.UID) + if err := p.store.updateStatus(ctx, cr); err != nil { + return commitRequestRequeueNow, ignoreConflict(err) + } + attach.GitTargetUID = cr.Status.GitTargetUID + } + if time.Since(cr.CreationTimestamp.Time) >= resolveRequestWindow(cr.Spec).resolveTimeout() { + // Past the bound only the withdraw is sent: an attach sent now would sit behind it on the + // worker's FIFO and could register the request after it was cancelled. + return c.withdraw(ctx, p, cr, attach) + } + result, resolved, err := c.finalizer.ServiceCommitRequest(ctx, attach) + if err != nil || !resolved { + if err != nil { + p.log.V(1).Info("CommitRequest attach not yet serviceable; will retry", + "name", client.ObjectKeyFromObject(cr), "err", err.Error()) + } + return c.recordPhase(ctx, p, cr, result.Phase, result.Held) + } + if result.Err != nil { + p.log.Error(result.Err, "CommitRequest finalize failed", "name", client.ObjectKeyFromObject(cr), + "gitTarget", cr.GitTargetKey().String()) + } + return c.decide(ctx, p, cr, commitRequestOutcome{result: result, err: result.Err}, true) +} + +// withdraw handles a claimed request past its safety window. A request the worker holds keeps polling +// with no bound of its own: only the worker can say how it ends. Any other request is withdrawn, and +// fails only on the worker's answer, so a request reported failed can never be committed afterwards. +func (c *commitRequestCore) withdraw( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, + attach git.AttachCommitRequest, +) (time.Duration, error) { + result, resolved, err := c.finalizer.WithdrawCommitRequest(ctx, attach) + if err != nil || !resolved { + if err != nil { + p.log.V(1).Info("CommitRequest withdraw not yet serviceable; will retry", + "name", client.ObjectKeyFromObject(cr), "err", err.Error()) + } + return c.recordPhase(ctx, p, cr, result.Phase, result.Held) + } + if !errors.Is(result.Err, git.ErrCommitRequestWithdrawn) { + // The worker resolved it on its own before the withdraw: report that outcome. + return c.decide(ctx, p, cr, commitRequestOutcome{result: result, err: result.Err}, true) + } + message := resolveTimeoutMessage + switch { + case errors.Is(result.Err, git.ErrBranchWorkerStopped): + message = git.ErrBranchWorkerStopped.Error() + case result.Phase == git.PhaseWaitingForWorker: + message = noWorkerTimeoutMessage + } + p.log.Info("CommitRequest was not acted on within the safety window; withdrawn and failed closed", + "name", client.ObjectKeyFromObject(cr), "reason", message) + // A worker's answer carries the branch, and only then is the request's GitTarget known. + return c.decide(ctx, p, cr, commitRequestOutcome{err: errors.New(message), + fallbackTarget: result.Branch == ""}, true) +} + +// refuseOnceUnheld fails a request that a replacement of its GitTarget refuses. The worker that may +// hold it answers first: its outcome is reported, and a request it holds is waited for. The refusal is +// written only once the request is withdrawn, or nothing held it. +func (c *commitRequestCore) refuseOnceUnheld( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, + attach git.AttachCommitRequest, refusal error, +) (time.Duration, error) { + result, resolved, err := c.finalizer.WithdrawCommitRequest(ctx, attach) + if err != nil || !resolved { + return c.recordPhase(ctx, p, cr, result.Phase, result.Held) + } + outcome := commitRequestOutcome{result: result, err: result.Err} + if errors.Is(result.Err, git.ErrCommitRequestWithdrawn) { + outcome = commitRequestOutcome{err: refusal} + } + return c.decide(ctx, p, cr, outcome, true) +} + +// recordPhase reports the phase the worker says the request is in, and polls again. It writes only on +// a change; a conflict is left to the next poll, which reads the request again. +func (c *commitRequestCore) recordPhase( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, + phase git.CommitRequestPhase, held string, +) (time.Duration, error) { + if progressRecorded(cr, phase, held) { + return commitRequestPollInterval, nil + } + markCommitRequestProgressing(cr, phase, held) + if err := p.store.updateStatus(ctx, cr); err != nil { + return commitRequestPollInterval, ignoreConflict(err) + } + return commitRequestPollInterval, nil +} + +// refusePrunedGitTargetRef fails a request whose spec names no GitTarget, which can only be one +// thing: an object stored before spec.targetRef was renamed to spec.gitTargetRef, whose old value the +// apiserver stopped serving the moment the new CRDs landed. Admission refuses an empty name on every +// path, so nothing else can produce it. It is terminal, because the spec is immutable: the message +// names the only fix there is. +func (c *commitRequestCore) refusePrunedGitTargetRef( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, +) error { + if len(cr.Status.Conditions) > 0 { + return nil + } + _, err := c.decide(ctx, p, cr, commitRequestOutcome{pruned: true, fallbackTarget: true}, false) + return err +} + +// commitRequestStreams holds, for each audit route a CommitRequest's creator is looked up on, the fact +// stream of commitrequests there, so the fact follower reads those audit facts at all. Without it the +// lookup would wait out the grace and name nobody. A stream is acquired the first time its route is +// needed and held from then on: a stream followed late is replayed from the TTL horizon, so the first +// request on a route still finds its fact. +type commitRequestStreams struct { + set *queue.FactStreamSet + mu sync.Mutex + held map[string]func() +} + +// ensure acquires the stream of commitrequests on route, once. +func (s *commitRequestStreams) ensure(route string) { + if s == nil || s.set == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if _, held := s.held[route]; held { + return + } + if s.held == nil { + s.held = map[string]func(){} + } + s.held[route] = s.set.Acquire(queue.FactStreamKeyFor(route, commitRequestGVR().GroupResource())) +} + +// ignoreConflict drops a conflict: the next step reads the request again and decides from that. +func ignoreConflict(err error) error { + if apierrors.IsConflict(err) { + return nil + } + return err +} diff --git a/internal/controller/commitrequest_core_test.go b/internal/controller/commitrequest_core_test.go new file mode 100644 index 000000000..bda11e6bd --- /dev/null +++ b/internal/controller/commitrequest_core_test.go @@ -0,0 +1,602 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "sync" + "testing" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/queue" + "github.com/ConfigButler/gitops-reverser/internal/watch" +) + +// fakeAuthors answers every author lookup alike, after delay, and records the queries. +type fakeAuthors struct { + user git.UserInfo + outcome git.AttributionOutcome + delay time.Duration + + mu sync.Mutex + queries []watch.AuthorQuery +} + +func (f *fakeAuthors) ResolveAuthor(_ context.Context, q watch.AuthorQuery) (git.UserInfo, git.AttributionOutcome) { + f.mu.Lock() + f.queries = append(f.queries, q) + f.mu.Unlock() + time.Sleep(f.delay) + return f.user, f.outcome +} + +func (f *fakeAuthors) lookups() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.queries) +} + +// jane is an author the audit facts name. +func jane() *fakeAuthors { + return &fakeAuthors{user: git.UserInfo{Username: "jane@example.com"}, outcome: git.AttributionResolved} +} + +// controlPath is the control-cluster path over c. +func controlPath(c client.Client) commitRequestPath { + return commitRequestPath{store: typedCommitRequestStore{reader: c, writer: c}, deletes: true, log: logr.Discard()} +} + +// watcherPath is the watch of KubeProvider default/name, over c: a kubeconfig that points back at the +// control cluster, so it sees the same requests the control-cluster path does. +func watcherPath(c client.Client, name string, deletes bool) commitRequestPath { + return commitRequestPath{ + provider: types.NamespacedName{Namespace: "default", Name: name}, + store: typedCommitRequestStore{reader: c, writer: c}, + deletes: deletes, + log: logr.Discard(), + } +} + +// kubeconfigProvider is a KubeProvider with a kubeconfig, which serves CommitRequests when serving is +// true. +func kubeconfigProvider(name string, serving bool) *configv1alpha3.KubeProvider { + provider := &configv1alpha3.KubeProvider{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: configv1alpha3.KubeProviderSpec{SecretRef: &meta.SecretKeyReference{Name: name}}, + } + if serving { + provider.Spec.CommitRequests = configv1alpha3.CommitRequestsReconcile + } + return provider +} + +// mirroringTarget is the GitTarget team-a-config, mirroring through provider. Through a provider +// other than the in-cluster one, it admits the requests of every namespace of the cluster it mirrors. +func mirroringTarget(provider string) *configv1alpha3.GitTarget { + target := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "default"}, + Spec: configv1alpha3.GitTargetSpec{KubeProviderRef: meta.NamespacedObjectReference{Name: provider}}, + } + if provider != localProvider { + target.Spec.AllowedCommitRequests = []configv1alpha3.GitTargetCommitRequestGrant{ + {SourceNamespaces: []configv1alpha3.SourceNamespace{"*"}}, + } + } + return target +} + +func committed() *fakeFinalizer { + return &fakeFinalizer{ + resolved: true, result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "c0ffee", Branch: "main"}, + } +} + +func serveStep(t *testing.T, core *commitRequestCore, p commitRequestPath, name string) time.Duration { + t.Helper() + after, err := core.serve(context.Background(), p, types.NamespacedName{Namespace: "default", Name: name}) + require.NoError(t, err) + return after +} + +func commitRequestReady(t *testing.T, c client.Client, name string) *metav1.Condition { + t.Helper() + cr := fetchCommitRequest(t, c, name) + return apimeta.FindStatusCondition(cr.Status.Conditions, ConditionTypeReady) +} + +// A request on the control cluster may be seen by the control-cluster path and by a kubeconfig watcher +// pointing back at the same cluster. The GitTarget's KubeProvider alone picks which one serves it; the +// other writes nothing. A request that leaves the namespace out is the control cluster's form, and a +// watcher never picks it up. +func TestCommitRequestCore_OverlappingObserversOfOneCluster(t *testing.T) { + for _, tc := range []struct { + name string + provider string + namespaceSet bool + wantServedBy string + wantNoService bool + }{ + {name: "in-cluster GitTarget, namespace omitted", provider: localProvider, wantServedBy: "control"}, + {name: "in-cluster GitTarget, namespace set", provider: localProvider, namespaceSet: true, + wantServedBy: "control"}, + {name: "watched GitTarget, namespace set", provider: "edge", namespaceSet: true, wantServedBy: "edge"}, + {name: "watched GitTarget, namespace omitted", provider: "edge", wantNoService: true}, + } { + t.Run(tc.name, func(t *testing.T) { + cr := newCommitRequest("save") + if tc.namespaceSet { + cr.Spec.GitTargetRef.Namespace = "default" + } + c := newCommitRequestClient(t, nil, cr, mirroringTarget(tc.provider), kubeconfigProvider("edge", true)) + f := committed() + core := &commitRequestCore{reader: c, finalizer: f} + + // The watcher steps first, so a wrong owner would get there before the right one. + serveStep(t, core, watcherPath(c, "edge", false), "save") + if tc.wantServedBy == "control" { + assert.Empty(t, fetchCommitRequest(t, c, "save").Status.Conditions, "the watcher writes nothing") + } + serveStep(t, core, controlPath(c), "save") + + ready := commitRequestReady(t, c, "save") + if tc.wantNoService { + assert.Nil(t, ready, "nobody serves it: the known gap of a kubeconfig to the control cluster") + assert.Empty(t, f.calls) + return + } + require.NotNil(t, ready) + assert.Equal(t, crReasonCommitted, ready.Reason, "served by %s", tc.wantServedBy) + assert.Len(t, f.calls, 1, "served once") + }) + } +} + +// Two kubeconfig providers on one source cluster both watch a request; only the GitTarget's own one +// serves it, whichever sees it first. The other never refuses it. +func TestCommitRequestCore_TwoProvidersOnOneSourceCluster(t *testing.T) { + cr := newCommitRequest("save") + cr.Spec.GitTargetRef.Namespace = "default" + c := newCommitRequestClient(t, nil, cr, mirroringTarget("team-a"), + kubeconfigProvider("team-a", true), kubeconfigProvider("team-b", true)) + f := committed() + core := &commitRequestCore{reader: c, finalizer: f} + + serveStep(t, core, watcherPath(c, "team-b", true), "save") + assert.Empty(t, fetchCommitRequest(t, c, "save").Status.Conditions, "the other provider leaves it alone") + serveStep(t, core, watcherPath(c, "team-a", true), "save") + + ready := commitRequestReady(t, c, "save") + require.NotNil(t, ready) + assert.Equal(t, crReasonCommitted, ready.Reason) + assert.Len(t, f.calls, 1) +} + +// A control-cluster request for a GitTarget that mirrors another cluster is refused when that +// KubeProvider serves no CommitRequests: the request's author would come from the control cluster's +// audit and its edits' from the other one's. One that serves them may point back here, so the request +// is left to its watch. +func TestCommitRequestCore_AGitTargetMirroringAnotherCluster(t *testing.T) { + t.Run("its provider serves no CommitRequests", func(t *testing.T) { + c := newCommitRequestClient(t, nil, newCommitRequest("save"), mirroringTarget("edge"), + kubeconfigProvider("edge", false)) + f := committed() + serveStep(t, &commitRequestCore{reader: c, finalizer: f}, controlPath(c), "save") + + ready := commitRequestReady(t, c, "save") + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, crReasonNotAllowed, ready.Reason) + assert.Contains(t, ready.Message, "create the CommitRequest there") + assert.Empty(t, f.calls) + }) + t.Run("its provider serves them", func(t *testing.T) { + c := newCommitRequestClient(t, nil, newCommitRequest("save"), mirroringTarget("edge"), + kubeconfigProvider("edge", true)) + f := committed() + after := serveStep(t, &commitRequestCore{reader: c, finalizer: f}, controlPath(c), "save") + + assert.Nil(t, commitRequestReady(t, c, "save"), "left alone") + assert.Empty(t, f.calls) + assert.Equal(t, commitRequestUnownedRecheck, after) + }) +} + +// staleStore serves its first read from a snapshot taken earlier, so a path decides from a version +// another path has since written over, as two processes racing would. +type staleStore struct { + typedCommitRequestStore + + snapshot *configv1alpha3.CommitRequest +} + +func (s *staleStore) get(ctx context.Context, key types.NamespacedName) (*configv1alpha3.CommitRequest, error) { + if snapshot := s.snapshot; snapshot != nil { + s.snapshot = nil + return snapshot.DeepCopy(), nil + } + return s.typedCommitRequestStore.get(ctx, key) +} + +// A GitTarget created during the attach wait races the GitTargetNotFound an observer decided before it +// existed. When the owner's claim lands first, the observer's write conflicts and it stops. +func TestCommitRequestCore_TheClaimWinsOverAStaleGitTargetNotFound(t *testing.T) { + cr := newCommitRequest("save") + cr.Spec.GitTargetRef.Namespace = "default" + cr.CreationTimestamp = metav1.NewTime(time.Now().Add(-time.Minute)) + c := newCommitRequestClient(t, nil, cr) + stale := fetchCommitRequest(t, c, "save") + f := &fakeFinalizer{} // the save goes on + serveStep(t, &commitRequestCore{reader: c, finalizer: f}, controlPath(c), "save") + require.Len(t, f.calls, 1, "the owner claimed the request and attached it") + + // The observer read the request, and found no GitTarget, before the owner's claim. + noTarget := newCommitRequestClient(t, nil, &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "elsewhere", Namespace: "default"}, + }) + observer := watcherPath(c, "edge", false) + observer.store = &staleStore{typedCommitRequestStore: typedCommitRequestStore{reader: c, writer: c}, + snapshot: &stale} + serveStep(t, &commitRequestCore{reader: noTarget, finalizer: &fakeFinalizer{}}, observer, "save") + + ready := commitRequestReady(t, c, "save") + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionUnknown, ready.Status, "the claim stands") + assert.Equal(t, crReasonSaving, ready.Reason) +} + +// When the GitTargetNotFound lands first, the owner's claim, decided from the version before it, +// conflicts: the owner reads the outcome and stops, and neither attaches nor commits. +func TestCommitRequestCore_AGitTargetNotFoundWinsOverALateClaim(t *testing.T) { + cr := newCommitRequest("save") + cr.Spec.GitTargetRef.Namespace = "default" + cr.CreationTimestamp = metav1.NewTime(time.Now().Add(-time.Minute)) + noTarget := newCommitRequestClient(t, nil, cr, &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "elsewhere", Namespace: "default"}, + }) + stale := fetchCommitRequest(t, noTarget, "save") + serveStep(t, &commitRequestCore{reader: noTarget, finalizer: &fakeFinalizer{}}, + watcherPath(noTarget, "edge", false), "save") + ready := commitRequestReady(t, noTarget, "save") + require.NotNil(t, ready) + require.Equal(t, crReasonGitTargetNotFound, ready.Reason) + + // The GitTarget appears; the owner read the request before the outcome was written. + withTarget := newCommitRequestClient(t, nil, mirroringTarget(localProvider)) + owner := controlPath(noTarget) + owner.store = &staleStore{typedCommitRequestStore: typedCommitRequestStore{reader: noTarget, writer: noTarget}, + snapshot: &stale} + f := committed() + serveStep(t, &commitRequestCore{reader: withTarget, finalizer: f}, owner, "save") + + assert.Empty(t, f.calls, "the owner never attaches a request another path finished") + ready = commitRequestReady(t, noTarget, "save") + require.NotNil(t, ready) + assert.Equal(t, crReasonGitTargetNotFound, ready.Reason) +} + +// GitTargetNotFound is written only after attachTimeout, and only into an empty status. +func TestCommitRequestCore_GitTargetNotFoundWaitsAndNeverOverwritesAClaim(t *testing.T) { + fresh := newCommitRequest("fresh") + fresh.Spec.GitTargetRef.Name = "missing" + claimed := withInProgress(newCommitRequest("claimed")) + claimed.Spec.GitTargetRef.Name = "missing" + claimed.CreationTimestamp = metav1.NewTime(time.Now().Add(-time.Minute)) + c := newCommitRequestClient(t, nil, fresh, claimed, mirroringTarget(localProvider)) + core := &commitRequestCore{reader: c, finalizer: &fakeFinalizer{}} + + after := serveStep(t, core, controlPath(c), "fresh") + assert.Positive(t, after) + assert.Empty(t, fetchCommitRequest(t, c, "fresh").Status.Conditions, "within attachTimeout nothing is written") + + serveStep(t, core, controlPath(c), "claimed") + ready := commitRequestReady(t, c, "claimed") + require.NotNil(t, ready) + assert.NotEqual(t, crReasonGitTargetNotFound, ready.Reason, "a claimed request is never refused for it") +} + +// A status conflict after the worker produced an outcome re-applies the outcome onto the fresh object; +// the save is not restarted. +func TestCommitRequestCore_AConflictAfterTheOutcomeReappliesIt(t *testing.T) { + conflicts := 1 + fns := interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, sub string, obj client.Object, + opts ...client.SubResourceUpdateOption) error { + cr, ok := obj.(*configv1alpha3.CommitRequest) + if ok && conflicts > 0 && commitRequestIsTerminal(cr) { + conflicts-- + return apierrors.NewConflict(commitRequestGVR().GroupResource(), cr.Name, nil) + } + return c.SubResource(sub).Update(ctx, obj, opts...) + }, + } + c := newCommitRequestClient(t, &fns, withInProgress(newCommitRequest("save"))) + f := committed() + serveStep(t, &commitRequestCore{reader: c, finalizer: f}, controlPath(c), "save") + + assert.Len(t, f.calls, 1, "the worker is asked once") + ready := commitRequestReady(t, c, "save") + require.NotNil(t, ready) + assert.Equal(t, crReasonCommitted, ready.Reason) + assert.Equal(t, "c0ffee", fetchCommitRequest(t, c, "save").Status.Commit) +} + +// After a restart, the owner finds its own claim and resumes the save with the author it recorded, +// without looking the author up again: the claim moved the resourceVersion the fact was keyed by. +func TestCommitRequestCore_ARestartResumesAClaimedSave(t *testing.T) { + cr := withInProgress(newCommitRequest("save")) + cr.Status.Author = "jane@example.com" + c := newCommitRequestClient(t, nil, cr) + authors := jane() + f := committed() + serveStep(t, &commitRequestCore{reader: c, finalizer: f, authors: authors}, controlPath(c), "save") + + require.Len(t, f.calls, 1) + assert.Equal(t, "jane@example.com", f.calls[0].Author) + assert.Equal(t, git.AttributionResolved, f.calls[0].Attribution) + assert.Zero(t, authors.lookups(), "the recorded author is used") + assert.Equal(t, crReasonCommitted, commitRequestReady(t, c, "save").Reason) +} + +// A request's author comes from its own audit fact, on the route of its GitTarget's KubeProvider, and +// is written in the claim, before anything is attached. The lookup is made with the fact stream of +// commitrequests acquired on that route. +func TestCommitRequestCore_TheAuthorComesFromAuditAndIsClaimedFirst(t *testing.T) { + cr := newCommitRequest("save") + cr.ResourceVersion = "" + c := newCommitRequestClient(t, nil, cr) + stored := fetchCommitRequest(t, c, "save") + authors := jane() + streams := queue.NewFactStreamSet() + var claimedFirst bool + f := &fakeFinalizer{} + core := &commitRequestCore{reader: c, finalizer: f, authors: authors, + streams: &commitRequestStreams{set: streams}} + path := controlPath(c) + path.store = authorCheckingStore{typedCommitRequestStore: typedCommitRequestStore{reader: c, writer: c}, + onWrite: func(cr *configv1alpha3.CommitRequest) { + if len(f.calls) == 0 && cr.Status.Author == "jane@example.com" { + claimedFirst = true + } + }} + serveStep(t, core, path, "save") + + require.Len(t, authors.queries, 1) + query := authors.queries[0] + assert.Equal(t, "default.local", query.AuditRoute, "the route of the GitTarget's KubeProvider") + assert.Equal(t, commitRequestGVR(), query.GVR) + assert.Equal(t, stored.ResourceVersion, query.ResourceVersion, "the version the request was created with") + assert.True(t, query.ExactCapable) + assert.Equal(t, 1, streams.Len(), "the stream of commitrequests on the route is followed") + assert.True(t, claimedFirst, "the author is written before the attach") + require.Len(t, f.calls, 1) + assert.Equal(t, "jane@example.com", f.calls[0].Author) + assert.Equal(t, "jane@example.com", fetchCommitRequest(t, c, "save").Status.Author) +} + +// authorCheckingStore calls onWrite with each status it writes. +type authorCheckingStore struct { + typedCommitRequestStore + + onWrite func(*configv1alpha3.CommitRequest) +} + +func (s authorCheckingStore) updateStatus(ctx context.Context, cr *configv1alpha3.CommitRequest) error { + s.onWrite(cr) + return s.typedCommitRequestStore.updateStatus(ctx, cr) +} + +// The request's audit fact may arrive late, within the grace, on a provider whose AuditFactsReceived is +// not True yet: the core waits for it whenever attribution is on, so the request is attributed. +func TestCommitRequestCore_ADelayedAuditFactStillNamesTheAuthor(t *testing.T) { + c := newCommitRequestClient(t, nil, newCommitRequest("save")) + authors := jane() + authors.delay = 50 * time.Millisecond + f := &fakeFinalizer{} + serveStep(t, &commitRequestCore{reader: c, finalizer: f, authors: authors}, controlPath(c), "save") + + require.Len(t, f.calls, 1) + assert.Equal(t, "jane@example.com", f.calls[0].Author) + assert.Equal(t, "jane@example.com", fetchCommitRequest(t, c, "save").Status.Author) +} + +// A request no audit fact names is refused on a provider that has received facts, since its fact went +// missing; on one that has not, it names nobody and saves writes that name nobody either. +func TestCommitRequestCore_ARequestNoFactNames(t *testing.T) { + unnamed := &fakeAuthors{outcome: git.AttributionUnresolved} + t.Run("on a provider that received facts", func(t *testing.T) { + provider := &configv1alpha3.KubeProvider{ + ObjectMeta: metav1.ObjectMeta{Name: localProvider, Namespace: "default"}, + Spec: configv1alpha3.KubeProviderSpec{InCluster: &configv1alpha3.KubeProviderInCluster{}}, + Status: configv1alpha3.KubeProviderStatus{Conditions: []metav1.Condition{{ + Type: KubeProviderConditionAuditFactsReceived, Status: metav1.ConditionTrue, Reason: "Received", + }}}, + } + c := newCommitRequestClient(t, nil, newCommitRequest("save"), mirroringTarget(localProvider), provider) + f := committed() + serveStep(t, &commitRequestCore{reader: c, finalizer: f, authors: unnamed}, controlPath(c), "save") + + assert.Empty(t, f.calls) + ready := commitRequestReady(t, c, "save") + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionTrue, ready.Status) + assert.Equal(t, crReasonAuthorUnresolved, ready.Reason) + }) + t.Run("on a provider that did not", func(t *testing.T) { + c := newCommitRequestClient(t, nil, newCommitRequest("save")) + f := &fakeFinalizer{} + serveStep(t, &commitRequestCore{reader: c, finalizer: f, authors: unnamed}, controlPath(c), "save") + + require.Len(t, f.calls, 1) + assert.Empty(t, f.calls[0].Author) + assert.Equal(t, git.AttributionUnresolved, f.calls[0].Attribution) + }) +} + +// factsArriveDuringTheWait names nobody, and while it waits the provider's first audit facts arrive: +// its AuditFactsReceived turns True. +type factsArriveDuringTheWait struct { + c client.Client + provider types.NamespacedName +} + +func (f factsArriveDuringTheWait) ResolveAuthor(ctx context.Context, _ watch.AuthorQuery) ( + git.UserInfo, git.AttributionOutcome, +) { + var provider configv1alpha3.KubeProvider + if err := f.c.Get(ctx, f.provider, &provider); err == nil { + apimeta.SetStatusCondition(&provider.Status.Conditions, metav1.Condition{ + Type: KubeProviderConditionAuditFactsReceived, Status: metav1.ConditionTrue, Reason: "Received", + }) + _ = f.c.Update(ctx, &provider) + } + return git.UserInfo{}, git.AttributionUnresolved +} + +// A provider whose first audit facts arrive while a request waits for its own is one that sends them: +// the request whose fact is still missing is refused, never attached to the unnamed window. +func TestCommitRequestCore_FactsArrivingDuringTheWaitRefuseAnUnnamedRequest(t *testing.T) { + c := newCommitRequestClient(t, nil, newCommitRequest("save")) + f := committed() + authors := factsArriveDuringTheWait{c: c, provider: types.NamespacedName{Namespace: "default", Name: localProvider}} + serveStep(t, &commitRequestCore{reader: c, finalizer: f, authors: authors}, controlPath(c), "save") + + assert.Empty(t, f.calls, "nothing is attached") + ready := commitRequestReady(t, c, "save") + require.NotNil(t, ready) + assert.Equal(t, crReasonAuthorUnresolved, ready.Reason) +} + +// A finished request is kept, its apply followed and its delete-after honored, only by the path that +// keeps it: the GitTarget's own, or the one that refused it because no path serves it. Another path that +// deletes finished requests leaves it alone, whatever delete-after it carries. Once its GitTarget is +// gone, no path owns it any more, and any path that deletes may. +func TestCommitRequestCore_RetentionIsTheKeepersAlone(t *testing.T) { + past := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339) + for _, tc := range []struct { + name string + target *configv1alpha3.GitTarget + providers []client.Object + path func(client.Client) commitRequestPath + goneTarget bool + wantDeleted bool + }{ + {name: "another provider's watcher that deletes", target: mirroringTarget("team-a"), + providers: []client.Object{kubeconfigProvider("team-a", true), kubeconfigProvider("team-b", true)}, + path: func(c client.Client) commitRequestPath { return watcherPath(c, "team-b", true) }}, + {name: "the control cluster, under a watcher pointing back at it", target: mirroringTarget("team-a"), + providers: []client.Object{kubeconfigProvider("team-a", true)}, path: controlPath}, + {name: "the owner that deletes", target: mirroringTarget("team-a"), + providers: []client.Object{kubeconfigProvider("team-a", true)}, + path: func(c client.Client) commitRequestPath { return watcherPath(c, "team-a", true) }, + wantDeleted: true}, + {name: "the control cluster that refused it", target: mirroringTarget("team-a"), + providers: []client.Object{kubeconfigProvider("team-a", false)}, path: controlPath, wantDeleted: true}, + {name: "any path that deletes, once the GitTarget is gone", target: mirroringTarget("team-a"), + providers: []client.Object{kubeconfigProvider("team-a", true), kubeconfigProvider("team-b", true)}, + path: func(c client.Client) commitRequestPath { return watcherPath(c, "team-b", true) }, + goneTarget: true, wantDeleted: true}, + } { + t.Run(tc.name, func(t *testing.T) { + cr := finishedWithDeleteAfter("save", past) + cr.Spec.GitTargetRef.Namespace = "default" + c := newCommitRequestClient(t, nil, append([]client.Object{cr, tc.target}, tc.providers...)...) + if tc.goneTarget { + require.NoError(t, c.Delete(context.Background(), tc.target)) + } + core := &commitRequestCore{reader: c, finalizer: committed(), ttl: time.Hour} + after := serveStep(t, core, tc.path(c), "save") + + assert.Equal(t, !tc.wantDeleted, commitRequestExists(t, c, "save")) + if !tc.wantDeleted { + assert.Equal(t, commitRequestUnownedRecheck, after, "looked at again, in case its owner changes") + } + }) + } +} + +// A finished request is stamped with delete-after and deleted once it has passed on a path that +// deletes finished requests, and left as it is on one that does not. Removing the annotation keeps it. +func TestCommitRequestCore_Retention(t *testing.T) { + for _, tc := range []struct { + name string + path func(client.Client) commitRequestPath + deletes bool + }{ + {name: "control cluster", path: controlPath, deletes: true}, + {name: "a watcher that deletes", path: func(c client.Client) commitRequestPath { + return watcherPath(c, "edge", true) + }, deletes: true}, + {name: "a watcher that does not", path: func(c client.Client) commitRequestPath { + return watcherPath(c, "edge", false) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + cr := newCommitRequest("save") + cr.Spec.GitTargetRef.Namespace = "default" + target := mirroringTarget("edge") + if tc.name == "control cluster" { + target = mirroringTarget(localProvider) + } + c := newCommitRequestClient(t, nil, cr, target, kubeconfigProvider("edge", true)) + core := &commitRequestCore{reader: c, finalizer: committed(), ttl: time.Hour} + serveStep(t, core, tc.path(c), "save") + + got := fetchCommitRequest(t, c, "save") + require.True(t, commitRequestIsTerminal(&got)) + _, stamped := got.Annotations[CommitRequestDeleteAfterAnnotation] + assert.Equal(t, tc.deletes, stamped) + + got.Annotations = map[string]string{ + CommitRequestDeleteAfterAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339), + } + require.NoError(t, c.Update(context.Background(), &got)) + serveStep(t, core, tc.path(c), "save") + var gone configv1alpha3.CommitRequest + err := c.Get(context.Background(), client.ObjectKeyFromObject(&got), &gone) + assert.Equal(t, tc.deletes, apierrors.IsNotFound(err), "deleted once delete-after passed") + }) + } + + t.Run("removing the annotation keeps the request", func(t *testing.T) { + c := newCommitRequestClient(t, nil, newCommitRequest("save")) + core := &commitRequestCore{reader: c, finalizer: committed(), ttl: time.Hour} + serveStep(t, core, controlPath(c), "save") + got := fetchCommitRequest(t, c, "save") + delete(got.Annotations, CommitRequestDeleteAfterAnnotation) + require.NoError(t, c.Update(context.Background(), &got)) + + assert.Zero(t, serveStep(t, core, controlPath(c), "save")) + fetchCommitRequest(t, c, "save") + }) +} + +// A request served by a watcher carries attach: Next and whenNothingToCommit: CommitEmpty to the +// worker, as one on the control cluster does. +func TestCommitRequestCore_AWatcherCarriesNextAndCommitEmpty(t *testing.T) { + cr := newCommitRequest("save") + cr.Spec.GitTargetRef.Namespace = "default" + cr.Spec.Window = &configv1alpha3.CommitRequestWindow{ + Attach: configv1alpha3.AttachNext, AttachTimeout: &metav1.Duration{Duration: time.Second}, + } + cr.Spec.WhenNothingToCommit = configv1alpha3.NothingToCommitCommitEmpty + c := newCommitRequestClient(t, nil, cr, mirroringTarget("edge"), kubeconfigProvider("edge", true)) + f := &fakeFinalizer{} + serveStep(t, &commitRequestCore{reader: c, finalizer: f}, watcherPath(c, "edge", false), "save") + + require.Len(t, f.calls, 1) + assert.Equal(t, configv1alpha3.AttachNext, f.calls[0].Attach) + assert.True(t, f.calls[0].CommitEmpty) +} diff --git a/internal/controller/commitrequest_decide.go b/internal/controller/commitrequest_decide.go new file mode 100644 index 000000000..cf07c63a5 --- /dev/null +++ b/internal/controller/commitrequest_decide.go @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "errors" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// commitRequestRequeueNow is the next step of a request whose write conflicted: it reads the request +// again and decides from that. +const commitRequestRequeueNow = 100 * time.Millisecond + +// errCommitRequestNotAllowed is a request its GitTarget does not admit, or one this path refuses. It is +// terminal: the spec is immutable, so a later grant reaches only a new request. +var errCommitRequestNotAllowed = errors.New("the CommitRequest is not allowed") + +// errGitTargetNotFound is a request whose GitTarget did not exist within its attachTimeout. +var errGitTargetNotFound = errors.New("the GitTarget does not exist") + +// authorUnresolvedMessage explains a request whose own audit event went missing on a cluster that sends +// them. +const authorUnresolvedMessage = "no audit event names who created this request, though its cluster sends " + + "audit events: its event went missing, so it saves nothing. Save again" + +// commitRequestOutcome is how one request ended, before it is written. +type commitRequestOutcome struct { + result git.FinalizeResult + err error + // authorUnresolved is a request no audit fact names on a cluster that sends them. + authorUnresolved bool + // pruned is a request whose spec names no GitTarget. + pruned bool + // fallbackTarget counts the outcome under empty GitTarget labels: the GitTarget was never resolved. + fallbackTarget bool +} + +// metricOutcome is the outcome's value on gitopsreverser_commit_requests_total. +func (o commitRequestOutcome) metricOutcome() string { + switch { + case o.pruned: + return crOutcomeFailed + case o.authorUnresolved: + return crOutcomeAuthorUnresolved + } + return commitRequestOutcomeValue(o.result, o.err) +} + +// apply writes the outcome onto cr's conditions. +func (o commitRequestOutcome) apply(cr *configbutleraiv1alpha3.CommitRequest) { + switch { + case o.pruned: + cr.Status.ObservedGeneration = cr.Generation + failCommitRequest(cr, crReasonGitTargetRefPruned, + "spec.gitTargetRef is empty: this request was created before spec.targetRef was renamed, "+ + "and its value was pruned by the upgrade. A CommitRequest spec is immutable, so this "+ + "one cannot be repaired — delete it and create a new one.") + case o.authorUnresolved: + cr.Status.ObservedGeneration = cr.Generation + rejectCommitRequest(cr, crReasonAuthorUnresolved, authorUnresolvedMessage) + default: + applyFinalizeResultToStatus(cr, o.result, o.err) + } +} + +// decide writes a request's outcome. afterClaim says the owner claimed the request: then a conflict +// re-applies the outcome onto the fresh object, unless that already holds one, and the save is never +// restarted. Before the claim a conflict on anything already written means another path got there +// first, and the next step decides again from what it wrote. +// +// An outcome after the claim is counted once, where it is settled, rather than per write attempt: the +// counter measures what the worker decided. One decided before the claim is counted only once written, +// since a path that loses the race decided nothing. +func (c *commitRequestCore) decide( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, + o commitRequestOutcome, afterClaim bool, +) (time.Duration, error) { + target := resolvedCommitRequestTarget(cr) + if o.fallbackTarget { + target = commitRequestTarget{} + } + if afterClaim { + recordCommitRequestOutcome(ctx, o.metricOutcome(), target) + } + current, written, err := c.writeOutcome(ctx, p, cr, o, afterClaim) + if err != nil || !written { + return commitRequestRequeueNow, err + } + if !afterClaim { + recordCommitRequestOutcome(ctx, o.metricOutcome(), target) + } + c.stampDeleteAfter(ctx, p, current) + readyReason, readyMessage := commitRequestReadyReason(current) + p.log.Info("CommitRequest finalized", + "name", client.ObjectKeyFromObject(current), + "ready", commitRequestConditionStatus(current, ConditionTypeReady), + "reason", readyReason, + "message", readyMessage, + "author", current.Status.Author, + "branch", current.Status.Branch, + "commit", current.Status.Commit, + "gitTarget", current.GitTargetKey().String(), + "age", time.Since(current.CreationTimestamp.Time).String()) + if followsApply(current) { + return commitRequestPollInterval, nil + } + return c.expireFinished(ctx, p, current) +} + +// writeOutcome writes o onto cr, re-reading it on a conflict. written is false when the fresh object +// already holds an outcome, or, before the claim, anything another path wrote; then the next step +// decides from that. +func (c *commitRequestCore) writeOutcome( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, + o commitRequestOutcome, afterClaim bool, +) (*configbutleraiv1alpha3.CommitRequest, bool, error) { + current := cr + for attempt := 1; ; attempt++ { + o.apply(current) + err := p.store.updateStatus(ctx, current) + if err == nil { + return current, true, nil + } + if !apierrors.IsConflict(err) || attempt == commitRequestStatusAttempts { + p.log.Error(err, "Failed to write CommitRequest status", "name", client.ObjectKeyFromObject(cr)) + return current, false, err + } + fresh, same, getErr := c.reread(ctx, p, current) + if getErr != nil || !same { + return current, false, getErr + } + if commitRequestIsTerminal(fresh) || (!afterClaim && len(fresh.Status.Conditions) > 0) { + return current, false, nil + } + current = fresh + } +} + +// serveFinished is the step of a request whose save ended: following the apply it asked for, and then +// its retention, both only on a path that keeps the request. +func (c *commitRequestCore) serveFinished( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, +) (time.Duration, error) { + if !followsApply(cr) && !expires(p, cr) { + return 0, nil + } + keeps, err := c.keeps(ctx, p, cr) + if err != nil { + return 0, err + } + if !keeps { + return commitRequestUnownedRecheck, nil + } + if followsApply(cr) { + return c.followApply(ctx, p, cr) + } + return c.expireFinished(ctx, p, cr) +} + +// keeps decides whether path p keeps a finished request: follows its apply and deletes it once its +// delete-after time has passed. That is the owner, or the path that refused the request because no path +// serves its GitTarget. A request whose GitTarget is gone, or that names none, has no owner left, and +// what is asked of it is true for every observer, so any path keeps it. +// +// Without this, a path that deletes finished requests would delete one that another path owns and +// keeps, whenever it carries a delete-after annotation: set by a person, or by an earlier mode. +func (c *commitRequestCore) keeps( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, +) (bool, error) { + if cr.Spec.GitTargetRef.Name == "" { + return true, nil + } + key, named := p.gitTargetKey(cr) + if !named { + return false, nil + } + target, found, err := c.gitTarget(ctx, key) + if err != nil || !found { + return !found, err + } + _, owner, refusal, err := c.owns(ctx, p, target) + return owner || refusal != "", err +} + +// stampDeleteAfter writes the delete-after annotation onto a request that has just finished, on a path +// that deletes finished requests. It runs only on the transition to an outcome, never on a later step +// of a finished request, which is what keeps requests someone un-stamped alone. +// +// A value that is already there wins: a creator may set its own delete-after, and a person may extend +// it in the moment between the outcome and this stamp. So the write is conditional on the version it +// was computed from, and a conflict reads the request again rather than overwriting. +// +// A write that still fails is logged, not retried: the request is then kept, the safe direction. +func (c *commitRequestCore) stampDeleteAfter( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, +) { + if !p.deletes || c.ttl <= 0 { + return + } + deleteAfter := time.Now().Add(c.ttl).UTC().Format(time.RFC3339) + current := cr + for attempt := 1; attempt <= commitRequestStatusAttempts; attempt++ { + if _, set := current.Annotations[CommitRequestDeleteAfterAnnotation]; set { + return + } + err := p.store.annotate(ctx, current, CommitRequestDeleteAfterAnnotation, deleteAfter) + if err == nil || apierrors.IsNotFound(err) { + return + } + if !apierrors.IsConflict(err) { + p.log.Error(err, "Failed to stamp delete-after on a finished CommitRequest; it will be kept", + "name", client.ObjectKeyFromObject(cr)) + return + } + fresh, same, getErr := c.reread(ctx, p, current) + if getErr != nil || !same { + return + } + current = fresh + } + p.log.Error(nil, "Gave up stamping delete-after after repeated conflicts; the request will be kept", + "name", client.ObjectKeyFromObject(cr)) +} + +// expireFinished deletes a finished request once the time in its delete-after annotation has passed, +// on a path that deletes finished requests, and otherwise comes back when it will have. No annotation +// means keep: the TTL only decides what is written at finish, so changing it never reaches back to a +// request already stamped, and a request whose annotation someone removed stays. +// +// The delete is conditional on the resourceVersion that was judged, so an edit to the annotation that +// races it fails the delete with a conflict, and the next step reads the edit. +func (c *commitRequestCore) expireFinished( + ctx context.Context, p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest, +) (time.Duration, error) { + if !expires(p, cr) { + return 0, nil + } + value := cr.Annotations[CommitRequestDeleteAfterAnnotation] + deleteAfter, parsed := parseDeleteAfter(value) + if !parsed { + // Kept rather than guessed at: an annotation a person mistyped is not consent to delete. + p.log.Info("CommitRequest delete-after annotation is not an RFC 3339 time; keeping the request", + "name", client.ObjectKeyFromObject(cr), "value", value) + return 0, nil + } + if remaining := time.Until(deleteAfter); remaining > 0 { + return remaining, nil + } + if err := p.store.remove(ctx, cr); err != nil { + if apierrors.IsConflict(err) { + return commitRequestRequeueNow, nil + } + return 0, client.IgnoreNotFound(err) + } + p.log.Info("Deleted finished CommitRequest past its delete-after time", + "name", client.ObjectKeyFromObject(cr), "deleteAfter", value) + return 0, nil +} + +// expires reports whether path p may delete cr at all: a path that deletes finished requests, and a +// request that carries a delete-after annotation and is not already being deleted. +func expires(p commitRequestPath, cr *configbutleraiv1alpha3.CommitRequest) bool { + _, ok := cr.Annotations[CommitRequestDeleteAfterAnnotation] + return p.deletes && ok && cr.DeletionTimestamp.IsZero() +} + +// parseDeleteAfter reads the annotation's RFC 3339 time, reporting false for anything else. +func parseDeleteAfter(value string) (time.Time, bool) { + t, err := time.Parse(time.RFC3339, value) + return t, err == nil +} diff --git a/internal/controller/commitrequest_finalize.go b/internal/controller/commitrequest_finalize.go index 313a31ae9..a75888978 100644 --- a/internal/controller/commitrequest_finalize.go +++ b/internal/controller/commitrequest_finalize.go @@ -4,6 +4,7 @@ package controller import ( "context" + "errors" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" @@ -17,22 +18,26 @@ import ( // CommitRequest condition reasons (CamelCase tokens surfaced on status.conditions). const ( - crReasonCommitted = "Committed" - crReasonNoWindow = "NoWindow" - crReasonWindowMismatch = "WindowMismatch" - crReasonAlreadyPresent = "AlreadyPresent" - crReasonFinalizeFailed = "FinalizeFailed" - crReasonUnexpectedOutcome = "UnexpectedOutcome" - crReasonAttributedFromAdmission = "AttributedFromAdmission" - crReasonCommitterFallback = "CommitterFallback" - crReasonAuthorCaptureDisabled = "AuthorCaptureDisabled" - crReasonGitTargetRefPruned = "GitTargetRefPruned" + crReasonCommitted = "Committed" + crReasonNoWindow = "NoWindow" + crReasonWindowMismatch = "WindowMismatch" + crReasonAlreadyPresent = "AlreadyPresent" + crReasonFinalizeFailed = "FinalizeFailed" + crReasonUnexpectedOutcome = "UnexpectedOutcome" + crReasonGitTargetRefPruned = "GitTargetRefPruned" + crReasonNotAllowed = "NotAllowed" + crReasonSaveInProgress = "SaveInProgress" + crReasonGitTargetNotFound = "GitTargetNotFound" + crReasonAuthorUnresolved = "AuthorUnresolved" + // crReasonSaving is the progress reason of a request just claimed, before the worker reports a + // phase. + crReasonSaving = "Saving" ) // The bounded `outcome` values on gitopsreverser_commit_requests_total. // // They are derived from git.FinalizeOutcome rather than from the condition reasons above: the -// FinalizeResult is what DECIDES, and the reasons include non-terminal and attribution-only values +// FinalizeResult is what DECIDES, and the reasons include non-terminal values // that have no business on a terminal counter. The set is closed, and outcomeFailed absorbs every // hard failure so an unmapped result can never go uncounted. const ( @@ -40,16 +45,19 @@ const ( crOutcomeNoWindow = "no_window" crOutcomeWindowMismatch = "window_mismatch" crOutcomeAlreadyPresent = "already_present" + crOutcomeSaveInProgress = "save_in_progress" crOutcomeFailed = "failed" + // crOutcomeAuthorUnresolved is a request no audit fact names, on a cluster that sends them. + crOutcomeAuthorUnresolved = "author_unresolved" ) -// commitRequestOutcome maps one finalize result to its counter value. +// commitRequestOutcomeValue maps one finalize result to its counter value. // // A finalize error outranks the outcome because FinalizeResult.Outcome is only set when Err is nil, // and an unrecognized outcome counts as failed for the same reason applyFinalizeResultToStatus // fails it: an empty or unknown outcome with no error is a bug, and a bug that increments nothing // is a bug nobody sees. -func commitRequestOutcome(result git.FinalizeResult, finalizeErr error) string { +func commitRequestOutcomeValue(result git.FinalizeResult, finalizeErr error) string { if finalizeErr != nil { return crOutcomeFailed } @@ -62,6 +70,8 @@ func commitRequestOutcome(result git.FinalizeResult, finalizeErr error) string { return crOutcomeWindowMismatch case git.FinalizeAlreadyPresent: return crOutcomeAlreadyPresent + case git.FinalizeSaveInProgress: + return crOutcomeSaveInProgress default: return crOutcomeFailed } @@ -88,10 +98,11 @@ type commitRequestTarget struct { // needed to bound the label — the plan's rule that metric recording must not block terminal status // on another failing call. // -// The namespace is the REQUEST's, not a field: a CommitRequest names a same-namespace GitTarget, -// which is the same assumption ServiceCommitRequest makes when it builds the object key. +// The key is CommitRequest.GitTargetKey, the GitTarget the attach names, which is the key +// ServiceCommitRequest gets. func resolvedCommitRequestTarget(cr *configv1alpha3.CommitRequest) commitRequestTarget { - return commitRequestTarget{namespace: cr.Namespace, name: cr.Spec.GitTargetRef.Name} + key := cr.GitTargetKey() + return commitRequestTarget{namespace: key.Namespace, name: key.Name} } // recordCommitRequestOutcome increments the terminal-outcome counter once. @@ -126,46 +137,6 @@ const alreadyPresentMessage = "the change already matches the remote, so no comm // notStalledMessage is the boilerplate message for a False Stalled condition. const notStalledMessage = "the CommitRequest is not stalled" -// commitRequestAttribution is the settled author decision threaded from the -// attribute step into the terminal status write so the AuthorAttributed condition can -// record whether the commit author was named. It is binary and always settled at -// first sight (present-or-never, §2) — there is no pending or timed-out state. -type commitRequestAttribution int - -const ( - // attributionUnset is the defensive zero value. It must never report a resolved - // actor when a caller neglected to initialize the attribution decision. - attributionUnset commitRequestAttribution = iota - // attributionFromAdmission means the submitter was captured at admission by the - // validate-operator-types webhook and named as the commit author. - attributionFromAdmission - // attributionCommitter means command-author capture ran but no admission author record - // exists, so the request does not claim an actor. - attributionCommitter - // attributionNotAttempted means command-author capture is switched off entirely (the - // validate-operator-types webhook is not running), so no submitter was ever sought. It is - // distinct from attributionCommitter, where capture DID run and found nothing. - attributionNotAttempted -) - -// gitOutcome projects the controller's attribution state onto the git layer's outcome, which -// is what the worker matches windows on. Keeping the projection here — rather than passing the -// controller's own enum down — means the git package never learns about admission records. -func (a commitRequestAttribution) gitOutcome() git.AttributionOutcome { - switch a { - case attributionUnset: - return git.AttributionNotAttempted - case attributionFromAdmission: - return git.AttributionResolved - case attributionCommitter: - return git.AttributionUnresolved - case attributionNotAttempted: - return git.AttributionNotAttempted - default: - return git.AttributionNotAttempted - } -} - // setCommitRequestCondition upserts a condition keyed by type, stamping it with // the request's generation so kstatus can tell a current status from a stale one. func setCommitRequestCondition( @@ -177,11 +148,11 @@ func setCommitRequestCondition( cr.Status.Conditions = upsertCondition(cr.Status.Conditions, conditionType, status, reason, message, cr.Generation) } -// Progress-condition messages, one per phase the worker reports. The author is settled -// synchronously at first sight (present-or-never, §2), so there is no "waiting for the author" -// phase. Until the worker has registered the request the reason is the generic Progressing. +// Progress-condition messages, one per phase the worker reports. The author is found before the +// claim, so there is no "waiting for the author" phase. From the claim until the worker has registered +// the request the reason is Saving. const ( - progressingMessage = "sent to the branch worker; waiting for it to register the request" + progressingMessage = "claimed; sent to the branch worker, waiting for it to register the request" waitingForWorkerMessage = "the GitTarget has no branch worker yet; waiting for its reconcile to start one" waitingForWindowMessage = "registered with the worker; waiting for a matching commit window to attach to" collectingWindowMessage = "attached to a commit window; collecting writes until its timers close it" @@ -206,15 +177,15 @@ func progressFor(phase git.CommitRequestPhase, held string) (string, string) { } return string(phase), waitingForPushMessage default: - return ReasonProgressing, progressingMessage + return crReasonSaving, progressingMessage } } -// setCommitRequestProgress stamps the four progress conditions (Ready=False, +// setCommitRequestProgress stamps the four progress conditions (Ready=Unknown, // Reconciling=True, Stalled=False, Pushed=Unknown) with one unifying reason, so a // single wait reads consistently across them. func setCommitRequestProgress(cr *configv1alpha3.CommitRequest, reason, message string) { - setCommitRequestCondition(cr, ConditionTypeReady, metav1.ConditionFalse, reason, message) + setCommitRequestCondition(cr, ConditionTypeReady, metav1.ConditionUnknown, reason, message) setCommitRequestCondition(cr, ConditionTypeReconciling, metav1.ConditionTrue, reason, message) setCommitRequestCondition(cr, ConditionTypeStalled, metav1.ConditionFalse, reason, notStalledMessage) setCommitRequestCondition(cr, ConditionTypePushed, metav1.ConditionUnknown, reason, pushPendingMessage) @@ -224,62 +195,34 @@ func setCommitRequestProgress(cr *configv1alpha3.CommitRequest, reason, message // (Reconciling=True). An empty phase is the generic Progressing: sent, not yet confirmed. func markCommitRequestProgressing( cr *configv1alpha3.CommitRequest, - attribution commitRequestAttribution, phase git.CommitRequestPhase, held string, ) { cr.Status.ObservedGeneration = cr.Generation - setCommitRequestAttributed(cr, attribution) reason, message := progressFor(phase, held) setCommitRequestProgress(cr, reason, message) } -// setCommitRequestAttributed records the settled, binary author decision on the -// AuthorAttributed condition. A False reason distinguishes an absent record -// (CommitterFallback) from capture disabled (AuthorCaptureDisabled). Neither is a failure or -// affects Ready; the request claims no actor. -func setCommitRequestAttributed(cr *configv1alpha3.CommitRequest, attribution commitRequestAttribution) { - switch attribution { - case attributionUnset: - setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, - crReasonAuthorCaptureDisabled, - "command-author capture state was unset; the request can attach only to a window with no named actor") - case attributionFromAdmission: - setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionTrue, - crReasonAttributedFromAdmission, - "the submitter was captured at admission and named as the commit author") - case attributionCommitter: - // Capture RAN and found nothing. Distinct from attributionNotAttempted below, whose - // message this used to carry — reporting "the webhook is not configured" for a - // webhook that is configured and simply missed sends operators to the wrong place. - setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, - crReasonCommitterFallback, - "no admission author record was found for this request; it can attach only to a "+ - "window with no named actor") - case attributionNotAttempted: - setCommitRequestCondition(cr, ConditionTypeAuthorAttributed, metav1.ConditionFalse, - crReasonAuthorCaptureDisabled, - "command-author capture is disabled (the validate-operator-types webhook is not "+ - "configured); the request can attach only to a window with no named actor") - } -} - -// applyFinalizeResultToStatus maps a FinalizeResult (or a finalize error) and the -// settled attribution onto a CommitRequest's terminal conditions. Ready is the -// summary; Reconciling/Stalled carry kstatus; Attributed records the author -// decision; Pushed records whether a commit reached the remote. +// applyFinalizeResultToStatus maps a FinalizeResult (or a finalize error) onto a +// CommitRequest's terminal conditions. Ready is the summary; Reconciling/Stalled +// carry kstatus; Pushed records whether a commit reached the remote. func applyFinalizeResultToStatus( cr *configv1alpha3.CommitRequest, result git.FinalizeResult, finalizeErr error, - attribution commitRequestAttribution, ) { cr.Status.ObservedGeneration = cr.Generation cr.Status.Branch = result.Branch - setCommitRequestAttributed(cr, attribution) if finalizeErr != nil { - failCommitRequest(cr, crReasonFinalizeFailed, finalizeErr.Error()) + reason := crReasonFinalizeFailed + switch { + case errors.Is(finalizeErr, errCommitRequestNotAllowed): + reason = crReasonNotAllowed + case errors.Is(finalizeErr, errGitTargetNotFound): + reason = crReasonGitTargetNotFound + } + failCommitRequest(cr, reason, finalizeErr.Error()) return } @@ -302,6 +245,10 @@ func applyFinalizeResultToStatus( case git.FinalizeAlreadyPresent: // The change already matches the remote, so the commit was dropped. resolveNothingToCommit(cr, result, crReasonAlreadyPresent, alreadyPresentMessage) + case git.FinalizeSaveInProgress: + // Refused, not failed by the worker: the spec is immutable, so the caller saves again with a + // new request once the earlier one resolved. + failCommitRequest(cr, crReasonSaveInProgress, saveInProgressMessage) default: // An empty or unknown outcome with no error is a bug, not a benign // rejection; fail it so it is not silently hidden. @@ -356,6 +303,13 @@ func commitRequestIsTerminal(cr *configv1alpha3.CommitRequest) bool { conditionIsTrue(cr.Status.Conditions, ConditionTypeStalled) } +// commitRequestClaimed reports whether a request's owner claimed it and its save has not ended: +// Ready=Unknown. +func commitRequestClaimed(cr *configv1alpha3.CommitRequest) bool { + ready := apimeta.FindStatusCondition(cr.Status.Conditions, ConditionTypeReady) + return ready != nil && ready.Status == metav1.ConditionUnknown && !commitRequestIsTerminal(cr) +} + // commitRequestConditionStatus returns the status of one condition as a loggable // string, or "" when the condition is absent. func commitRequestConditionStatus(cr *configv1alpha3.CommitRequest, conditionType string) string { diff --git a/internal/controller/commitrequest_kstatus_test.go b/internal/controller/commitrequest_kstatus_test.go index c84221dc6..1602c7962 100644 --- a/internal/controller/commitrequest_kstatus_test.go +++ b/internal/controller/commitrequest_kstatus_test.go @@ -24,7 +24,7 @@ func TestCommitRequestKstatusContract(t *testing.T) { wantMsg string }{ { - name: "missing admission author in the close-delay wait (AuthorAttributed=False is not a failure)", + name: "in the close-delay wait", conds: []map[string]interface{}{ conditionMap(ConditionTypeReady, "False", string(git.PhaseWaitingForWindow), waitingForWindowMessage), conditionMap( @@ -34,23 +34,6 @@ func TestCommitRequestKstatusContract(t *testing.T) { waitingForWindowMessage, ), conditionMap(ConditionTypeStalled, "False", string(git.PhaseWaitingForWindow), notStalledMessage), - conditionMap(ConditionTypeAuthorAttributed, "False", crReasonCommitterFallback, "no admission record"), - conditionMap(ConditionTypePushed, "Unknown", string(git.PhaseWaitingForWindow), pushPendingMessage), - }, - wantStatus: kstatus.InProgressStatus, - }, - { - name: "admission-attributed in the close-delay wait", - conds: []map[string]interface{}{ - conditionMap(ConditionTypeReady, "False", string(git.PhaseWaitingForWindow), waitingForWindowMessage), - conditionMap( - ConditionTypeReconciling, - "True", - string(git.PhaseWaitingForWindow), - waitingForWindowMessage, - ), - conditionMap(ConditionTypeStalled, "False", string(git.PhaseWaitingForWindow), notStalledMessage), - conditionMap(ConditionTypeAuthorAttributed, "True", crReasonAttributedFromAdmission, "from admission"), conditionMap(ConditionTypePushed, "Unknown", string(git.PhaseWaitingForWindow), pushPendingMessage), }, wantStatus: kstatus.InProgressStatus, @@ -61,7 +44,6 @@ func TestCommitRequestKstatusContract(t *testing.T) { conditionMap(ConditionTypeReady, "True", crReasonCommitted, "closed, committed, and pushed"), conditionMap(ConditionTypeReconciling, "False", crReasonCommitted, "closed, committed, and pushed"), conditionMap(ConditionTypeStalled, "False", crReasonCommitted, notStalledMessage), - conditionMap(ConditionTypeAuthorAttributed, "True", crReasonAttributedFromAdmission, "from admission"), conditionMap(ConditionTypePushed, "True", ReasonSucceeded, "pushed"), }, wantStatus: kstatus.CurrentStatus, diff --git a/internal/controller/commitrequest_outcome_metric_test.go b/internal/controller/commitrequest_outcome_metric_test.go index 7e8b7de77..5133f57de 100644 --- a/internal/controller/commitrequest_outcome_metric_test.go +++ b/internal/controller/commitrequest_outcome_metric_test.go @@ -19,6 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -60,13 +61,13 @@ func TestCommitRequestMetric_TwoTargetsAreDistinguishable(t *testing.T) { for _, target := range []string{"team-a-config", "team-b-config"} { cr := newCommitRequest("save-" + target) - cr.Spec.GitTargetRef = meta.LocalObjectReference{Name: target} + cr.Spec.GitTargetRef = meta.NamespacedObjectReference{Name: target} c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{ result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-"+target) } @@ -90,7 +91,7 @@ func TestCommitRequestMetric_RepeatedSavesReuseOneSeries(t *testing.T) { result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, name) } @@ -108,7 +109,7 @@ func TestCommitRequestMetric_UnresolvableTargetTakesTheFallback(t *testing.T) { require.NoError(t, err) cr := newCommitRequest("save-bogus") - cr.Spec.GitTargetRef = meta.LocalObjectReference{Name: "no-such-target"} + cr.Spec.GitTargetRef = meta.NamespacedObjectReference{Name: "no-such-target"} // Older than the resolve timeout, so the poll gives up on this pass rather than requeueing. cr.CreationTimestamp = metav1.NewTime(time.Now().Add(-2 * resolveRequestWindow(cr.Spec).resolveTimeout())) c := newCommitRequestClient(t, nil, cr) @@ -119,7 +120,7 @@ func TestCommitRequestMetric_UnresolvableTargetTakesTheFallback(t *testing.T) { withdrawResult: git.FinalizeResult{Err: git.ErrCommitRequestWithdrawn}, withdrawResolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-bogus") @@ -159,7 +160,7 @@ func TestCommitRequestOutcome_MapsEveryFinalizeResult(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, commitRequestOutcome(tc.result, tc.err)) + assert.Equal(t, tc.want, commitRequestOutcomeValue(tc.result, tc.err)) }) } } @@ -201,7 +202,7 @@ func TestCommitRequestMetric_EveryOutcomeIsReachableThroughReconcile(t *testing. c := newCommitRequestClient(t, nil, newCommitRequest("save-outcome")) f := &fakeFinalizer{result: tc.result, err: tc.err, resolved: true} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-outcome") @@ -242,7 +243,7 @@ func TestCommitRequestMetric_ConflictRetriesIncrementOnce(t *testing.T) { result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-conflict") @@ -275,10 +276,15 @@ func TestCommitRequestMetric_FailedStatusWriteThenRedeliveryCountsTwice(t *testi result: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} - reconcileCommitRequest(t, r, "save-lost") - reconcileCommitRequest(t, r, "save-lost") + // The failed write is returned, so the request is reconciled again. + for range 2 { + _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: client.ObjectKey{Namespace: "default", Name: "save-lost"}, + }) + require.Error(t, err) + } value, ok := outcomeCount(t, reader, crOutcomeCommitted) require.True(t, ok) @@ -296,13 +302,13 @@ func TestCommitRequestMetric_PrunedGitTargetRefCountsFailed(t *testing.T) { pruned := &configv1alpha3.CommitRequest{ ObjectMeta: metav1.ObjectMeta{Name: "save-pruned", Namespace: "default", UID: types.UID("uid-save-pruned")}, Spec: configv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{Name: ""}, + GitTargetRef: meta.NamespacedObjectReference{Name: ""}, Message: "save: pruned", }, } c := newCommitRequestClient(t, nil, pruned) f := &fakeFinalizer{} - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-pruned") @@ -326,7 +332,7 @@ func TestCommitRequestMetric_RecreatedRequestCountsPerIncarnation(t *testing.T) result: git.FinalizeResult{Outcome: git.FinalizeWindowMismatch, Branch: "main"}, resolved: true, } - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f} reconcileCommitRequest(t, r, "save-again") @@ -361,16 +367,16 @@ func TestCommitRequestMetric_AMissingWorkerKeepsTheTargetLabels(t *testing.T) { }, } fresh := newCommitRequest("save-fresh") - fresh.Spec.GitTargetRef = meta.LocalObjectReference{Name: target.Name} + fresh.Spec.GitTargetRef = meta.NamespacedObjectReference{Name: target.Name} fresh.CreationTimestamp = metav1.Now() stale := newCommitRequest("save-stale") - stale.Spec.GitTargetRef = meta.LocalObjectReference{Name: target.Name} + stale.Spec.GitTargetRef = meta.NamespacedObjectReference{Name: target.Name} // Older than the resolve timeout, so the poll gives up on this pass rather than requeueing. stale.CreationTimestamp = metav1.NewTime(time.Now().Add(-2 * resolveRequestWindow(stale.Spec).resolveTimeout())) c := newCommitRequestClient(t, nil, fresh, stale, target, gitBranchOn("team-a-provider", "default", "main")) workers := git.NewWorkerManager(c, logr.Discard(), git.BranchWorkerLimits{}, itypes.SensitiveResourcePolicy{}) router := watch.NewEventRouter(workers, nil, c, logr.Discard()) - r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: router, AuthorLookup: attributedAlice()} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: router} // Inside the safety window: pending, and the status says why. res := reconcileCommitRequest(t, r, fresh.Name) diff --git a/internal/controller/commitrequest_server.go b/internal/controller/commitrequest_server.go new file mode 100644 index 000000000..dbc0ba8b3 --- /dev/null +++ b/internal/controller/commitrequest_server.go @@ -0,0 +1,726 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + k8swatch "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/queue" + "github.com/ConfigButler/gitops-reverser/internal/watch" +) + +// The source-cluster path of a CommitRequest: a request created on a cluster a KubeProvider with a +// kubeconfig connects to, next to the edits it saves. CommitRequestServer watches the CommitRequests of +// every KubeProvider whose spec.commitRequests is one of the Reconcile values, through that provider's +// own identity, and hands each one to commitRequestCore, as the CommitRequestReconciler does on the +// control cluster. It writes status, and with ReconcileAndDeleteFinished deletes finished requests, +// through the same identity; it never writes the CRD. +// +// A provider watches only the namespaces it lends the GitTargets that mirror through it, in +// spec.allowedGitTargets, so a credential scoped to those namespaces is enough. What it watches, and +// whether the cluster serves the CRD at all, is reported to the KubeProvider reconciler, which writes +// it as the CommitRequestsServed condition and status.commitRequests. + +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch + +const ( + // commitRequestProviderResync is how often the server looks again at which KubeProviders to serve, + // what they lend, and whether their clusters serve the CRD. + commitRequestProviderResync = 10 * time.Second + // commitRequestReconnect is the wait before a watch of a cluster's CommitRequests opens again. + commitRequestReconnect = 10 * time.Second + // commitRequestEventBuffer is how many report changes wait for the KubeProvider reconciler. + commitRequestEventBuffer = 256 + // commitRequestRelist ends a watch this often, so every request is looked at again: one waiting for + // its delete-after time, or left to another path. + commitRequestRelist = 5 * time.Minute +) + +// CommitRequestsServed reasons, on a KubeProvider. +const ( + ReasonCommitRequestsWatching = "Watching" + ReasonCommitRequestsNothingToWatch = "NothingToWatch" + ReasonCommitRequestsCRDNotInstalled = "CRDNotInstalled" + ReasonCommitRequestsVersionNotServed = "VersionNotServed" + ReasonCommitRequestsForbidden = "Forbidden" + ReasonCommitRequestsUnreachable = "Unreachable" +) + +// commitRequestCRDFix is the fix for a cluster that serves no usable CommitRequest CRD. +const commitRequestCRDFix = "apply it with kubectl apply --server-side -f commitrequest-crd.yaml, from the " + + "release of GitOps Reverser this cluster is served by" + +// CommitRequestClusters reaches a source cluster through a KubeProvider's identity, with clients built +// for the call, and says which version of the credential each was built from. *watch.Manager satisfies +// it. +type CommitRequestClusters interface { + SourceDynamicClient(ctx context.Context, clusterID string) (dynamic.Interface, string, error) + SourceDiscovery(ctx context.Context, clusterID string) (discovery.DiscoveryInterface, string, error) +} + +// CommitRequestsReport is what the server last found of one KubeProvider's CommitRequests: the +// CommitRequestsServed condition, and the namespaces it watches. +type CommitRequestsReport struct { + Status metav1.ConditionStatus + Reason string + Message string + Namespaces []string +} + +// CommitRequestServer serves the CommitRequests of every KubeProvider that reconciles them. It runs on +// the leader only, as every status writer does. +type CommitRequestServer struct { + // Client reads KubeProviders, GitTargets and Namespaces on the control cluster. + Client client.Client + Clusters CommitRequestClusters + Finalizer CommitRequestFinalizer + // Authors finds a request's creator from the source cluster's audit facts; nil with attribution + // off. FactStreams is the set the fact follower reads; each watcher acquires the stream of + // commitrequests on its provider's route. + Authors watch.AuthorResolver + FactStreams *queue.FactStreamSet + // TTL is how long a finished request is kept on a provider that deletes them. + TTL time.Duration + Log logr.Logger + + mu sync.Mutex + watchers map[k8stypes.NamespacedName]*commitRequestWatcher + reports map[k8stypes.NamespacedName]CommitRequestsReport + events chan event.GenericEvent + streams commitRequestStreams +} + +// commitRequestWatcher is the watch of one KubeProvider's CommitRequests, in the namespaces it lends. +type commitRequestWatcher struct { + provider k8stypes.NamespacedName + config string + namespaces []string + path commitRequestPath + cancel context.CancelFunc + // serving holds the UIDs of the requests a goroutine serves now. + serving sync.Map + + mu sync.Mutex + // forbidden are the namespaces whose list or watch was denied, and undeletable those where a + // finished request could not be deleted. unreachable is why the cluster's client could not be built. + forbidden map[string]bool + undeletable map[string]bool + unreachable error +} + +// CommitRequestsEvents names a KubeProvider whose report changed, so its reconcile writes it at once. +func (s *CommitRequestServer) CommitRequestsEvents() <-chan event.GenericEvent { + s.mu.Lock() + defer s.mu.Unlock() + return s.eventsLocked() +} + +func (s *CommitRequestServer) eventsLocked() chan event.GenericEvent { + if s.events == nil { + s.events = make(chan event.GenericEvent, commitRequestEventBuffer) + } + return s.events +} + +// CommitRequestsReport is the last report for provider, and false before the first one, or while the +// provider serves no CommitRequests. +func (s *CommitRequestServer) CommitRequestsReport(provider k8stypes.NamespacedName) (CommitRequestsReport, bool) { + s.mu.Lock() + defer s.mu.Unlock() + report, ok := s.reports[provider] + return report, ok +} + +// Start runs the server until ctx ends. +func (s *CommitRequestServer) Start(ctx context.Context) error { + ticker := time.NewTicker(commitRequestProviderResync) + defer ticker.Stop() + for { + s.syncProviders(ctx) + select { + case <-ctx.Done(): + s.stopAll() + return nil + case <-ticker.C: + } + } +} + +// syncProviders runs a watcher for every KubeProvider that reconciles its CommitRequests, in the +// namespaces it lends now, and stops the rest; and reports each one. +func (s *CommitRequestServer) syncProviders(ctx context.Context) { + var providers configbutleraiv1alpha3.KubeProviderList + if err := s.Client.List(ctx, &providers); err != nil { + s.Log.V(1).Info("Cannot list KubeProviders for CommitRequests; will retry", "err", err.Error()) + return + } + var targets configbutleraiv1alpha3.GitTargetList + if err := s.Client.List(ctx, &targets); err != nil { + s.Log.V(1).Info("Cannot list GitTargets for CommitRequests; will retry", "err", err.Error()) + return + } + serving := map[k8stypes.NamespacedName]bool{} + for i := range providers.Items { + provider := &providers.Items[i] + if provider.DeletionTimestamp != nil || !provider.ServesCommitRequests() { + continue + } + serving[provider.Key()] = true + s.syncProvider(ctx, provider, targets.Items) + } + s.mu.Lock() + defer s.mu.Unlock() + for key, watcher := range s.watchers { + if !serving[key] { + watcher.cancel() + delete(s.watchers, key) + } + } + for key := range s.reports { + if !serving[key] { + delete(s.reports, key) + s.notifyLocked(key) + } + } +} + +// syncProvider brings one provider's watcher in line with what it lends and with its cluster's CRD, +// and reports it. +func (s *CommitRequestServer) syncProvider( + ctx context.Context, provider *configbutleraiv1alpha3.KubeProvider, targets []configbutleraiv1alpha3.GitTarget, +) { + key := provider.Key() + namespaces, err := s.lentNamespaces(ctx, provider, targets) + if err != nil { + s.Log.V(1).Info("Cannot read what a KubeProvider lends; will retry", "kubeProvider", key, + "err", err.Error()) + return + } + report, credential := s.probeCRD(ctx, key) + if report.Reason == "" { + report = s.runWatcher(ctx, provider, namespaces, credential) + } else { + s.stopWatcher(key) + } + report.Namespaces = namespaces + s.mu.Lock() + defer s.mu.Unlock() + if s.reports == nil { + s.reports = map[k8stypes.NamespacedName]CommitRequestsReport{} + } + if previous, ok := s.reports[key]; ok && commitRequestsReportsEqual(previous, report) { + return + } + s.reports[key] = report + s.notifyLocked(key) +} + +// lentNamespaces are the namespaces provider lends, in spec.allowedGitTargets, to the GitTargets that +// mirror through it: sorted, and "*" alone when any grant lends every namespace. +func (s *CommitRequestServer) lentNamespaces( + ctx context.Context, provider *configbutleraiv1alpha3.KubeProvider, targets []configbutleraiv1alpha3.GitTarget, +) ([]string, error) { + var lent []string + for i := range targets { + target := &targets[i] + if target.DeletionTimestamp != nil || target.KubeProviderKey() != provider.Key() { + continue + } + labels, err := namespaceLabelsOf(ctx, s.Client, target.Namespace) + if err != nil { + return nil, err + } + scope, err := provider.AdmitsGitTarget(client.ObjectKeyFromObject(target), labels) + if err != nil || !scope.Admitted { + continue + } + lent = append(lent, scope.SourceNamespaces...) + } + if slices.Contains(lent, "*") { + return []string{"*"}, nil + } + sort.Strings(lent) + return slices.Compact(lent), nil +} + +// probeCRD reports, by discovery, a cluster that serves no CommitRequest CRD in v1alpha3, or that +// could not be read; an empty reason means the CRD is there. It also returns the version of the +// credential it was read with. +func (s *CommitRequestServer) probeCRD( + ctx context.Context, provider k8stypes.NamespacedName, +) (CommitRequestsReport, string) { + dc, credential, err := s.Clusters.SourceDiscovery(ctx, watch.SourceClusterID(provider)) + if err != nil { + return unreachableCommitRequests(err), "" + } + return commitRequestCRDReport(dc), credential +} + +// commitRequestCRDReport reads, through dc, whether its cluster serves commitrequests in +// configbutler.ai/v1alpha3. +func commitRequestCRDReport(dc discovery.DiscoveryInterface) CommitRequestsReport { + groups, err := dc.ServerGroups() + if err != nil { + return unreachableCommitRequests(err) + } + gv := commitRequestGVR().GroupVersion() + var group *metav1.APIGroup + for i := range groups.Groups { + if groups.Groups[i].Name == gv.Group { + group = &groups.Groups[i] + } + } + notInstalled := CommitRequestsReport{Status: metav1.ConditionFalse, Reason: ReasonCommitRequestsCRDNotInstalled, + Message: "the cluster serves no commitrequests.configbutler.ai CRD: " + commitRequestCRDFix} + if group == nil { + return notInstalled + } + if !slices.ContainsFunc(group.Versions, func(v metav1.GroupVersionForDiscovery) bool { + return v.Version == gv.Version + }) { + return CommitRequestsReport{Status: metav1.ConditionFalse, Reason: ReasonCommitRequestsVersionNotServed, + Message: fmt.Sprintf("the commitrequests.configbutler.ai CRD does not serve %s: %s", + gv.Version, commitRequestCRDFix)} + } + resources, err := dc.ServerResourcesForGroupVersion(gv.String()) + if err != nil { + if apierrors.IsNotFound(err) { + return notInstalled + } + return unreachableCommitRequests(err) + } + if !slices.ContainsFunc(resources.APIResources, func(r metav1.APIResource) bool { + return r.Name == commitRequestGVR().Resource + }) { + return notInstalled + } + return CommitRequestsReport{} +} + +func unreachableCommitRequests(err error) CommitRequestsReport { + return CommitRequestsReport{Status: metav1.ConditionUnknown, Reason: ReasonCommitRequestsUnreachable, + Message: "the cluster's API could not be read: " + err.Error()} +} + +// runWatcher starts, restarts or keeps provider's watcher for namespaces, and reports how it goes. A +// changed credential restarts it, so its watches and the status it writes move to the new credential +// together, rather than keep the client it started with. +func (s *CommitRequestServer) runWatcher( + ctx context.Context, provider *configbutleraiv1alpha3.KubeProvider, namespaces []string, credential string, +) CommitRequestsReport { + key := provider.Key() + if len(namespaces) == 0 { + s.stopWatcher(key) + return CommitRequestsReport{Status: metav1.ConditionTrue, Reason: ReasonCommitRequestsNothingToWatch, + Message: "no GitTarget mirroring through this KubeProvider is lent a namespace in spec.allowedGitTargets"} + } + deletes := provider.DeletesFinishedCommitRequests() + config := fmt.Sprintf("%s|%t|%s|%s", provider.AuditRoute(), deletes, strings.Join(namespaces, ","), credential) + s.mu.Lock() + if s.watchers == nil { + s.watchers = map[k8stypes.NamespacedName]*commitRequestWatcher{} + } + watcher, running := s.watchers[key] + if running && watcher.config != config { + watcher.cancel() + running = false + } + if !running { + watchCtx, cancel := context.WithCancel(ctx) + watcher = &commitRequestWatcher{provider: key, config: config, namespaces: namespaces, cancel: cancel, + forbidden: map[string]bool{}, undeletable: map[string]bool{}} + s.watchers[key] = watcher + go s.run(watchCtx, watcher, provider.AuditRoute(), deletes) + } + s.mu.Unlock() + return watcher.report(deletes) +} + +func (s *CommitRequestServer) stopWatcher(key k8stypes.NamespacedName) { + s.mu.Lock() + defer s.mu.Unlock() + if watcher, ok := s.watchers[key]; ok { + watcher.cancel() + delete(s.watchers, key) + } +} + +func (s *CommitRequestServer) stopAll() { + s.mu.Lock() + defer s.mu.Unlock() + for key, watcher := range s.watchers { + watcher.cancel() + delete(s.watchers, key) + } +} + +// notifyLocked names provider on the events channel, without blocking: a full channel is caught up by +// the provider's steady requeue. +func (s *CommitRequestServer) notifyLocked(provider k8stypes.NamespacedName) { + select { + case s.eventsLocked() <- event.GenericEvent{Object: &configbutleraiv1alpha3.KubeProvider{ + ObjectMeta: metav1.ObjectMeta{Namespace: provider.Namespace, Name: provider.Name}, + }}: + default: + } +} + +// report is the CommitRequestsServed condition of a running watcher. +func (w *commitRequestWatcher) report(deletes bool) CommitRequestsReport { + w.mu.Lock() + defer w.mu.Unlock() + verbs := "get, list and watch on commitrequests, and update on commitrequests/status" + if deletes { + verbs += ", and patch and delete on commitrequests" + } + if w.unreachable != nil { + return unreachableCommitRequests(w.unreachable) + } + if len(w.forbidden) > 0 { + return CommitRequestsReport{Status: metav1.ConditionFalse, Reason: ReasonCommitRequestsForbidden, + Message: fmt.Sprintf("the credential may not list or watch CommitRequests in %s: grant it %s, "+ + "in the configbutler.ai group", describeNamespaces(sortedKeys(w.forbidden)), verbs)} + } + message := "watching CommitRequests in every namespace" + if !slices.Equal(w.namespaces, []string{"*"}) { + message = fmt.Sprintf("watching CommitRequests in %d namespaces: %s", len(w.namespaces), + strings.Join(w.namespaces, ", ")) + } + if len(w.undeletable) > 0 { + message += fmt.Sprintf("; finished requests in %s cannot be deleted and stay: grant delete on "+ + "commitrequests there", describeNamespaces(sortedKeys(w.undeletable))) + } + return CommitRequestsReport{Status: metav1.ConditionTrue, Reason: ReasonCommitRequestsWatching, Message: message} +} + +func (w *commitRequestWatcher) mark(set map[string]bool, namespace string, on bool) { + w.mu.Lock() + defer w.mu.Unlock() + if on { + set[namespace] = true + } else { + delete(set, namespace) + } +} + +// run watches one provider's CommitRequests until its context ends: one watch per lent namespace, or one +// across the cluster. +func (s *CommitRequestServer) run( + ctx context.Context, watcher *commitRequestWatcher, route string, deletes bool, +) { + log := s.Log.WithValues("kubeProvider", watcher.provider.String()) + log.Info("Serving the CommitRequests of a KubeProvider", "namespaces", watcher.namespaces) + if s.FactStreams != nil { + release := s.FactStreams.Acquire(queue.FactStreamKeyFor(route, commitRequestGVR().GroupResource())) + defer release() + } + dc, err := s.clusterClient(ctx, log, watcher) + if err != nil { + return + } + resource := dc.Resource(commitRequestGVR()) + watcher.path = commitRequestPath{ + provider: watcher.provider, + store: dynamicCommitRequestStore{resource: resource, undeletable: watcher.markUndeletable}, + deletes: deletes, + log: log, + } + var wg sync.WaitGroup + for _, namespace := range watcher.namespaces { + if namespace == "*" { + namespace = metav1.NamespaceAll + } + wg.Add(1) + go func() { + defer wg.Done() + s.watchNamespace(ctx, log, watcher, resource, namespace) + }() + } + wg.Wait() +} + +func (w *commitRequestWatcher) markUndeletable(namespace string) { + w.mark(w.undeletable, namespace, true) +} + +// clusterClient builds the client of the watcher's cluster, retrying until it can or ctx ends; until +// then the watcher reports the cluster unreachable. +func (s *CommitRequestServer) clusterClient( + ctx context.Context, log logr.Logger, watcher *commitRequestWatcher, +) (dynamic.Interface, error) { + for { + dc, _, err := s.Clusters.SourceDynamicClient(ctx, watch.SourceClusterID(watcher.provider)) + watcher.mu.Lock() + watcher.unreachable = err + watcher.mu.Unlock() + if err == nil { + return dc, nil + } + log.Info("Cannot reach the KubeProvider's cluster for CommitRequests; will retry", "err", err.Error()) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(commitRequestReconnect): + } + } +} + +// watchNamespace lists and watches one namespace's CommitRequests, or every namespace's, reconnecting +// after a failure, until ctx ends. +func (s *CommitRequestServer) watchNamespace( + ctx context.Context, log logr.Logger, watcher *commitRequestWatcher, + resource dynamic.NamespaceableResourceInterface, namespace string, +) { + for ctx.Err() == nil { + err := s.session(ctx, watcher, resource.Namespace(namespace), namespaceLabel(namespace)) + if apierrors.IsForbidden(err) { + watcher.mark(watcher.forbidden, namespaceLabel(namespace), true) + } + if err != nil && ctx.Err() == nil { + log.Info("The watch of CommitRequests ended; reconnecting", "namespace", namespaceLabel(namespace), + "err", err.Error()) + } + select { + case <-ctx.Done(): + case <-time.After(commitRequestReconnect): + } + } +} + +// session lists the requests and watches them, until the watch ends. A list that succeeds clears a +// denial reported for the namespace, so a Role granted since is reported at once. +func (s *CommitRequestServer) session( + ctx context.Context, watcher *commitRequestWatcher, resource dynamic.ResourceInterface, namespace string, +) error { + list, err := resource.List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + watcher.mark(watcher.forbidden, namespace, false) + for i := range list.Items { + s.consider(ctx, watcher, &list.Items[i]) + } + timeout := int64(commitRequestRelist / time.Second) + w, err := resource.Watch(ctx, metav1.ListOptions{ + ResourceVersion: list.GetResourceVersion(), AllowWatchBookmarks: true, TimeoutSeconds: &timeout, + }) + if err != nil { + return err + } + defer w.Stop() + for { + select { + case <-ctx.Done(): + return nil + case ev, ok := <-w.ResultChan(): + if !ok { + return nil + } + switch ev.Type { + case k8swatch.Added, k8swatch.Modified: + if u, isObject := ev.Object.(*unstructured.Unstructured); isObject { + s.consider(ctx, watcher, u) + } + case k8swatch.Error: + return apierrors.FromObject(ev.Object) + case k8swatch.Deleted, k8swatch.Bookmark: + } + } + } +} + +// consider serves a request no goroutine serves yet: it takes steps until the core has nothing due +// before the next relist looks at the request again. +func (s *CommitRequestServer) consider( + ctx context.Context, + watcher *commitRequestWatcher, + u *unstructured.Unstructured, +) { + if u.GetDeletionTimestamp() != nil { + return + } + if _, serving := watcher.serving.LoadOrStore(u.GetUID(), struct{}{}); serving { + return + } + key := k8stypes.NamespacedName{Namespace: u.GetNamespace(), Name: u.GetName()} + core := s.core() + go func() { + defer watcher.serving.Delete(u.GetUID()) + for ctx.Err() == nil { + after, err := core.serve(ctx, watcher.path, key) + if err != nil { + watcher.path.log.V(1).Info("A CommitRequest step failed; will retry", "commitRequest", key, + "err", err.Error()) + after = commitRequestPollInterval + } + if after == 0 || after >= commitRequestRelist { + return + } + select { + case <-ctx.Done(): + case <-time.After(after): + } + } + }() +} + +func (s *CommitRequestServer) core() *commitRequestCore { + s.mu.Lock() + s.streams.set = s.FactStreams + s.mu.Unlock() + return &commitRequestCore{ + reader: s.Client, finalizer: s.Finalizer, authors: s.Authors, streams: &s.streams, ttl: s.TTL, + } +} + +// dynamicCommitRequestStore reads and writes a source cluster's CommitRequests through a dynamic +// client. Every write carries the object's resourceVersion, so it is conditional on it. +type dynamicCommitRequestStore struct { + resource dynamic.NamespaceableResourceInterface + // undeletable is told the namespace of a finished request the credential may not delete. + undeletable func(namespace string) +} + +func (s dynamicCommitRequestStore) get( + ctx context.Context, key k8stypes.NamespacedName, +) (*configbutleraiv1alpha3.CommitRequest, error) { + u, err := s.resource.Namespace(key.Namespace).Get(ctx, key.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return commitRequestFromUnstructured(u) +} + +func (s dynamicCommitRequestStore) updateStatus(ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest) error { + u, err := commitRequestToUnstructured(cr) + if err != nil { + return err + } + written, err := s.resource.Namespace(cr.Namespace).UpdateStatus(ctx, u, metav1.UpdateOptions{}) + if err != nil { + return err + } + return commitRequestInto(written, cr) +} + +func (s dynamicCommitRequestStore) annotate( + ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest, key, value string, +) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{ + "resourceVersion": cr.ResourceVersion, "annotations": map[string]string{key: value}, + }}) + if err != nil { + return err + } + written, err := s.resource.Namespace(cr.Namespace).Patch(ctx, cr.Name, k8stypes.MergePatchType, patch, + metav1.PatchOptions{}) + if err != nil { + return err + } + return commitRequestInto(written, cr) +} + +func (s dynamicCommitRequestStore) remove(ctx context.Context, cr *configbutleraiv1alpha3.CommitRequest) error { + uid, resourceVersion := cr.UID, cr.ResourceVersion + err := s.resource.Namespace(cr.Namespace).Delete(ctx, cr.Name, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &resourceVersion}, + }) + if apierrors.IsForbidden(err) && s.undeletable != nil { + s.undeletable(cr.Namespace) + return nil + } + return err +} + +func commitRequestFromUnstructured(u *unstructured.Unstructured) (*configbutleraiv1alpha3.CommitRequest, error) { + var cr configbutleraiv1alpha3.CommitRequest + if err := commitRequestInto(u, &cr); err != nil { + return nil, err + } + return &cr, nil +} + +func commitRequestInto(u *unstructured.Unstructured, cr *configbutleraiv1alpha3.CommitRequest) error { + *cr = configbutleraiv1alpha3.CommitRequest{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, cr); err != nil { + return fmt.Errorf("read CommitRequest %s/%s: %w", u.GetNamespace(), u.GetName(), err) + } + return nil +} + +func commitRequestToUnstructured(cr *configbutleraiv1alpha3.CommitRequest) (*unstructured.Unstructured, error) { + object, err := runtime.DefaultUnstructuredConverter.ToUnstructured(cr) + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{Object: object} + u.SetGroupVersionKind(configbutleraiv1alpha3.GroupVersion.WithKind("CommitRequest")) + return u, nil +} + +// namespaceLabelsOf reads the labels of a control-cluster namespace; one that does not exist has none. +func namespaceLabelsOf(ctx context.Context, reader client.Reader, namespace string) (map[string]string, error) { + var ns corev1.Namespace + if err := reader.Get(ctx, client.ObjectKey{Name: namespace}, &ns); err != nil { + if apierrors.IsNotFound(err) { + return map[string]string{}, nil + } + return nil, fmt.Errorf("read Namespace %s: %w", namespace, err) + } + return ns.Labels, nil +} + +// namespaceLabel names a watched namespace in a message, "*" for every namespace. +func namespaceLabel(namespace string) string { + if namespace == metav1.NamespaceAll { + return "*" + } + return namespace +} + +// describeNamespaces names namespaces in a message: "namespace a" or "namespaces a, b", and "every +// namespace" for "*". +func describeNamespaces(namespaces []string) string { + switch { + case slices.Contains(namespaces, "*"): + return "every namespace" + case len(namespaces) == 1: + return "namespace " + namespaces[0] + } + return "namespaces " + strings.Join(namespaces, ", ") +} + +func sortedKeys(set map[string]bool) []string { + keys := make([]string, 0, len(set)) + for key := range set { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func commitRequestsReportsEqual(a, b CommitRequestsReport) bool { + return a.Status == b.Status && a.Reason == b.Reason && a.Message == b.Message && + slices.Equal(a.Namespaces, b.Namespaces) +} diff --git a/internal/controller/commitrequest_server_test.go b/internal/controller/commitrequest_server_test.go new file mode 100644 index 000000000..756f02e70 --- /dev/null +++ b/internal/controller/commitrequest_server_test.go @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "errors" + "slices" + "sync" + "testing" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/dynamic" + dynamicfake "k8s.io/client-go/dynamic/fake" + clienttesting "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// fakeDiscovery serves resources, and fails with err when it is set. +func fakeDiscovery(err error, resources ...*metav1.APIResourceList) discovery.DiscoveryInterface { + fake := &clienttesting.Fake{Resources: resources} + if err != nil { + fake.AddReactor("get", "group", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, err + }) + } + return &fakediscovery.FakeDiscovery{Fake: fake} +} + +func commitRequestResources(groupVersion string, names ...string) *metav1.APIResourceList { + list := &metav1.APIResourceList{GroupVersion: groupVersion} + for _, name := range names { + list.APIResources = append(list.APIResources, metav1.APIResource{Name: name, Namespaced: true}) + } + return list +} + +// The CRD's state is read by discovery alone: no permission on CRDs is needed, so a credential scoped to +// namespaces reports it too. Each reason that is not True says how to fix it. +func TestCommitRequestCRDReport(t *testing.T) { + for _, tc := range []struct { + name string + discovery discovery.DiscoveryInterface + wantReason string + wantFix string + }{ + {name: "no configbutler.ai group", discovery: fakeDiscovery(nil), + wantReason: ReasonCommitRequestsCRDNotInstalled, wantFix: "kubectl apply --server-side"}, + {name: "the group without commitrequests", + discovery: fakeDiscovery(nil, commitRequestResources("configbutler.ai/v1alpha3", "gittargets")), + wantReason: ReasonCommitRequestsCRDNotInstalled, wantFix: "commitrequest-crd.yaml"}, + {name: "another version only", + discovery: fakeDiscovery(nil, commitRequestResources("configbutler.ai/v1alpha2", "commitrequests")), + wantReason: ReasonCommitRequestsVersionNotServed, wantFix: "commitrequest-crd.yaml"}, + {name: "served", discovery: fakeDiscovery(nil, + commitRequestResources("configbutler.ai/v1alpha3", "commitrequests"))}, + {name: "unreadable", discovery: fakeDiscovery(errors.New("connection refused")), + wantReason: ReasonCommitRequestsUnreachable, wantFix: "connection refused"}, + } { + t.Run(tc.name, func(t *testing.T) { + report := commitRequestCRDReport(tc.discovery) + assert.Equal(t, tc.wantReason, report.Reason) + assert.Contains(t, report.Message, tc.wantFix) + }) + } +} + +// A provider watches the namespaces it lends the GitTargets that mirror through it, "*" alone once any +// grant lends every namespace. +func TestCommitRequestServer_LentNamespaces(t *testing.T) { + provider := kubeconfigProvider("edge", true) + provider.Spec.AllowedGitTargets = []configv1alpha3.KubeProviderGitTargetGrant{{ + ConsumerNamespaces: configv1alpha3.ConsumerNamespaces{From: configv1alpha3.GrantFromAll}, + SourceNamespaces: []configv1alpha3.ScopeNamespace{configv1alpha3.ConsumerNamespaceVariable, "shared"}, + }} + target := func(namespace, provider string) configv1alpha3.GitTarget { + return configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "t", Namespace: namespace}, + Spec: configv1alpha3.GitTargetSpec{KubeProviderRef: meta.NamespacedObjectReference{ + Name: provider, Namespace: "default", + }}, + } + } + c := newCommitRequestClient(t, nil) + s := &CommitRequestServer{Client: c} + + lent, err := s.lentNamespaces(context.Background(), provider, []configv1alpha3.GitTarget{ + target("team-b", "edge"), target("team-a", "edge"), target("team-c", "other"), + }) + require.NoError(t, err) + assert.Equal(t, []string{"shared", "team-a", "team-b"}, lent, "only its own GitTargets, sorted, once each") + + provider.Spec.AllowedGitTargets[0].SourceNamespaces = []configv1alpha3.ScopeNamespace{"*"} + lent, err = s.lentNamespaces(context.Background(), provider, []configv1alpha3.GitTarget{target("team-a", "edge")}) + require.NoError(t, err) + assert.Equal(t, []string{"*"}, lent) +} + +// A watcher reports what it watches, the namespaces it may not list or watch, and the ones where a +// finished request could not be deleted, which leaves serving True. +func TestCommitRequestWatcher_Report(t *testing.T) { + watcher := &commitRequestWatcher{namespaces: []string{"shop"}, forbidden: map[string]bool{}, + undeletable: map[string]bool{}} + report := watcher.report(true) + assert.Equal(t, metav1.ConditionTrue, report.Status) + assert.Equal(t, ReasonCommitRequestsWatching, report.Reason) + assert.Contains(t, report.Message, "1 namespaces: shop") + + watcher.markUndeletable("shop") + report = watcher.report(true) + assert.Equal(t, metav1.ConditionTrue, report.Status, "serving still works") + assert.Contains(t, report.Message, "cannot be deleted") + + watcher.mark(watcher.forbidden, "shop", true) + report = watcher.report(true) + assert.Equal(t, metav1.ConditionFalse, report.Status) + assert.Equal(t, ReasonCommitRequestsForbidden, report.Reason) + assert.Contains(t, report.Message, "namespace shop") + assert.Contains(t, report.Message, "patch and delete", "the verbs ReconcileAndDeleteFinished needs") + + every := &commitRequestWatcher{namespaces: []string{"*"}, forbidden: map[string]bool{}, + undeletable: map[string]bool{}} + assert.Contains(t, every.report(false).Message, "every namespace") + + every.unreachable = errors.New("no route to host") + report = every.report(false) + assert.Equal(t, ReasonCommitRequestsUnreachable, report.Reason) + assert.Contains(t, report.Message, "no route to host") +} + +// On a source cluster a request is admitted by a grant's sourceNamespaces, and only in a namespace the +// KubeProvider lends the GitTarget; on the control cluster by consumerNamespaces. Each refusal says +// which field applies. +func TestCommitRequestCore_AdmissionBySide(t *testing.T) { + t.Run("a source cluster request with no sourceNamespaces grant", func(t *testing.T) { + cr := newCommitRequest("save") + cr.Spec.GitTargetRef.Namespace = "default" + target := mirroringTarget("edge") + target.Spec.AllowedCommitRequests = nil + c := newCommitRequestClient(t, nil, cr, target, kubeconfigProvider("edge", true)) + f := committed() + serveStep(t, &commitRequestCore{reader: c, finalizer: f}, watcherPath(c, "edge", false), "save") + + assert.Empty(t, f.calls) + ready := commitRequestReady(t, c, "save") + require.NotNil(t, ready) + assert.Equal(t, crReasonNotAllowed, ready.Reason) + assert.Contains(t, ready.Message, "sourceNamespaces") + }) + t.Run("a source cluster request in a namespace that is not lent", func(t *testing.T) { + cr := newCommitRequest("save") + cr.Namespace = "elsewhere" + cr.Spec.GitTargetRef.Namespace = "default" + c := newCommitRequestClient(t, nil, cr, mirroringTarget("edge"), kubeconfigProvider("edge", true)) + f := committed() + _, err := (&commitRequestCore{reader: c, finalizer: f}).serve(context.Background(), + watcherPath(c, "edge", false), types.NamespacedName{Namespace: "elsewhere", Name: "save"}) + require.NoError(t, err) + + assert.Empty(t, f.calls) + var got configv1alpha3.CommitRequest + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Namespace: "elsewhere", Name: "save"}, &got)) + ready := apimeta.FindStatusCondition(got.Status.Conditions, ConditionTypeReady) + require.NotNil(t, ready) + assert.Equal(t, crReasonNotAllowed, ready.Reason) + assert.Contains(t, ready.Message, "does not lend namespace") + }) + t.Run("a control-cluster request no consumerNamespaces grant admits", func(t *testing.T) { + target := mirroringTarget(localProvider) + target.Spec.AllowedCommitRequests = []configv1alpha3.GitTargetCommitRequestGrant{ + {SourceNamespaces: []configv1alpha3.SourceNamespace{"*"}}, + } + c := newCommitRequestClient(t, nil, newCommitRequest("save"), target) + serveStep(t, &commitRequestCore{reader: c, finalizer: committed()}, controlPath(c), "save") + + ready := commitRequestReady(t, c, "save") + require.NotNil(t, ready) + assert.Equal(t, crReasonNotAllowed, ready.Reason) + assert.Contains(t, ready.Message, "consumerNamespaces") + }) +} + +// fakeClusters serves one source cluster: its dynamic client, and its discovery, under one credential. +type fakeClusters struct { + dynamic dynamic.Interface + discovery discovery.DiscoveryInterface +} + +func (f fakeClusters) SourceDynamicClient(context.Context, string) (dynamic.Interface, string, error) { + return f.dynamic, "v1", nil +} + +func (f fakeClusters) SourceDiscovery(context.Context, string) (discovery.DiscoveryInterface, string, error) { + return f.discovery, "v1", nil +} + +// rotatingClusters serves one source cluster whose credential the test rotates. Each credential reaches +// the cluster through a client of its own, so a client built from an old one sees none of what the new +// one does. +type rotatingClusters struct { + discovery discovery.DiscoveryInterface + + mu sync.Mutex + credential string + clients map[string]dynamic.Interface + // built are the credentials a client was built from, in order. + built []string +} + +func (r *rotatingClusters) rotate(credential string) { + r.mu.Lock() + defer r.mu.Unlock() + r.credential = credential +} + +func (r *rotatingClusters) SourceDynamicClient(context.Context, string) (dynamic.Interface, string, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.built = append(r.built, r.credential) + return r.clients[r.credential], r.credential, nil +} + +func (r *rotatingClusters) builtFrom() []string { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.built) +} + +func (r *rotatingClusters) SourceDiscovery(context.Context, string) (discovery.DiscoveryInterface, string, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.discovery, r.credential, nil +} + +// sourceCommitRequest is a CommitRequest as it is stored on a source cluster. +func sourceCommitRequest(namespace, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "configbutler.ai/v1alpha3", + "kind": "CommitRequest", + "metadata": map[string]any{ + "namespace": namespace, "name": name, "uid": "uid-" + name, "resourceVersion": "1", + "creationTimestamp": time.Now().UTC().Format(time.RFC3339), + }, + "spec": map[string]any{ + "gitTargetRef": map[string]any{"namespace": "default", "name": "team-a-config"}, + "message": "save: from the workspace", + }, + }} + return u +} + +// End to end through the server: a KubeProvider that reconciles its CommitRequests watches the namespace +// it lends, serves a request created there through the core, writes its status back on that cluster, +// and reports what it watches. With ReconcileAndDeleteFinished the finished request is stamped for +// deletion. +func TestCommitRequestServer_ServesARequestOnASourceCluster(t *testing.T) { + provider := kubeconfigProvider("edge", true) + provider.Spec.CommitRequests = configv1alpha3.CommitRequestsReconcileAndDeleteFinished + c := newCommitRequestClient(t, nil, mirroringTarget("edge"), provider) + gvr := commitRequestGVR() + source := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), + map[schema.GroupVersionResource]string{gvr: "CommitRequestList"}, sourceCommitRequest("default", "save")) + f := committed() + s := &CommitRequestServer{ + Client: c, Finalizer: f, TTL: time.Hour, Log: logr.Discard(), + Clusters: fakeClusters{dynamic: source, discovery: fakeDiscovery(nil, + commitRequestResources("configbutler.ai/v1alpha3", "commitrequests"))}, + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s.syncProviders(ctx) + + report, ok := s.CommitRequestsReport(provider.Key()) + require.True(t, ok) + assert.Equal(t, ReasonCommitRequestsWatching, report.Reason) + assert.Equal(t, []string{"default"}, report.Namespaces, "the namespace the provider lends its GitTarget") + select { + case ev := <-s.CommitRequestsEvents(): + assert.Equal(t, provider.Name, ev.Object.GetName(), "the KubeProvider is told its report changed") + default: + t.Fatal("no event for the changed report") + } + + require.Eventually(t, func() bool { + u, err := source.Resource(gvr).Namespace("default").Get(ctx, "save", metav1.GetOptions{}) + if err != nil { + return false + } + cr, convErr := commitRequestFromUnstructured(u) + if convErr != nil { + return false + } + ready := apimeta.FindStatusCondition(cr.Status.Conditions, ConditionTypeReady) + _, stamped := cr.Annotations[CommitRequestDeleteAfterAnnotation] + return ready != nil && ready.Reason == crReasonCommitted && stamped + }, 10*time.Second, 50*time.Millisecond, "the request is served on its own cluster, and stamped for deletion") + require.Len(t, f.calls, 1) + assert.Equal(t, git.AttributionNotAttempted, f.calls[0].Attribution, "attribution is off") + + provider.Spec.CommitRequests = configv1alpha3.CommitRequestsIgnore + require.NoError(t, c.Update(ctx, provider)) + s.syncProviders(ctx) + _, ok = s.CommitRequestsReport(provider.Key()) + assert.False(t, ok, "a provider that ignores CommitRequests reports nothing") +} + +// A rotated kubeconfig moves a provider's watcher to the new credential: its watches, and the status it +// writes, run on a client built from it, rather than on the one the watcher started with. +func TestCommitRequestServer_ARotatedCredentialMovesTheWatcher(t *testing.T) { + provider := kubeconfigProvider("edge", true) + c := newCommitRequestClient(t, nil, mirroringTarget("edge"), provider) + gvr := commitRequestGVR() + listKinds := map[schema.GroupVersionResource]string{gvr: "CommitRequestList"} + before := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), listKinds) + after := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), listKinds, + sourceCommitRequest("default", "save")) + clusters := &rotatingClusters{ + discovery: fakeDiscovery(nil, commitRequestResources("configbutler.ai/v1alpha3", "commitrequests")), + credential: "v1", + clients: map[string]dynamic.Interface{"v1": before, "v2": after}, + } + f := committed() + s := &CommitRequestServer{Client: c, Finalizer: f, Clusters: clusters, Log: logr.Discard()} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s.syncProviders(ctx) + report, ok := s.CommitRequestsReport(provider.Key()) + require.True(t, ok) + require.Equal(t, ReasonCommitRequestsWatching, report.Reason) + require.Eventually(t, func() bool { return len(clusters.builtFrom()) > 0 }, 10*time.Second, + 10*time.Millisecond, "the watcher runs on the first credential") + + clusters.rotate("v2") + s.syncProviders(ctx) + + require.Eventually(t, func() bool { + u, err := after.Resource(gvr).Namespace("default").Get(ctx, "save", metav1.GetOptions{}) + if err != nil { + return false + } + cr, convErr := commitRequestFromUnstructured(u) + if convErr != nil { + return false + } + ready := apimeta.FindStatusCondition(cr.Status.Conditions, ConditionTypeReady) + return ready != nil && ready.Reason == crReasonCommitted + }, 10*time.Second, 50*time.Millisecond, "the request is seen, and served, through the new credential") + assert.Equal(t, []string{"v1", "v2"}, clusters.builtFrom()) +} + +// A KubeProvider carries CommitRequestsServed and status.commitRequests only while it serves its +// cluster's CommitRequests. +func TestKubeProviderReconciler_CommitRequestsServed(t *testing.T) { + reporter := &fixedCommitRequestsReporter{report: CommitRequestsReport{ + Status: metav1.ConditionTrue, Reason: ReasonCommitRequestsWatching, Message: "watching", + Namespaces: []string{"shop"}, + }} + r := &KubeProviderReconciler{CommitRequests: reporter} + + provider := kubeconfigProvider("edge", true) + st := beginStatus(nil, nil, provider) + r.applyCommitRequestsServed(st, provider) + served := apimeta.FindStatusCondition(provider.Status.Conditions, KubeProviderConditionCommitRequestsServed) + require.NotNil(t, served) + assert.Equal(t, ReasonCommitRequestsWatching, served.Reason) + require.NotNil(t, provider.Status.CommitRequests) + assert.Equal(t, []string{"shop"}, provider.Status.CommitRequests.Namespaces) + + provider.Spec.CommitRequests = configv1alpha3.CommitRequestsIgnore + r.applyCommitRequestsServed(st, provider) + assert.Nil(t, apimeta.FindStatusCondition(provider.Status.Conditions, KubeProviderConditionCommitRequestsServed), + "removed while the provider ignores CommitRequests") + assert.Nil(t, provider.Status.CommitRequests) +} + +type fixedCommitRequestsReporter struct { + report CommitRequestsReport +} + +func (f *fixedCommitRequestsReporter) CommitRequestsReport(types.NamespacedName) (CommitRequestsReport, bool) { + return f.report, true +} + +func (f *fixedCommitRequestsReporter) CommitRequestsEvents() <-chan event.GenericEvent { return nil } diff --git a/internal/controller/commitrequest_ttl_test.go b/internal/controller/commitrequest_ttl_test.go index 5f84ba029..c35a1bd99 100644 --- a/internal/controller/commitrequest_ttl_test.go +++ b/internal/controller/commitrequest_ttl_test.go @@ -48,7 +48,7 @@ func committedFinalizer() *fakeFinalizer { func TestCommitRequestTTL_FinishingStampsTheDeletionTime(t *testing.T) { c := newCommitRequestClient(t, nil, newCommitRequest("save-now")) r := &CommitRequestReconciler{ - Client: c, APIReader: c, Finalizer: committedFinalizer(), AuthorLookup: attributedAlice(), + Client: c, APIReader: c, Finalizer: committedFinalizer(), TTL: 48 * time.Hour, } @@ -65,7 +65,7 @@ func TestCommitRequestTTL_FinishingStampsTheDeletionTime(t *testing.T) { func TestCommitRequestTTL_ZeroStampsNothing(t *testing.T) { c := newCommitRequestClient(t, nil, newCommitRequest("save-now")) r := &CommitRequestReconciler{ - Client: c, APIReader: c, Finalizer: committedFinalizer(), AuthorLookup: attributedAlice(), + Client: c, APIReader: c, Finalizer: committedFinalizer(), } reconcileCommitRequest(t, r, "save-now") @@ -128,7 +128,7 @@ func TestCommitRequestTTL_LeavesARequestInProgressAlone(t *testing.T) { c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{resolved: false} r := &CommitRequestReconciler{ - Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice(), TTL: time.Nanosecond, + Client: c, APIReader: c, Finalizer: f, TTL: time.Nanosecond, } res := reconcileCommitRequest(t, r, "save-pending") @@ -161,7 +161,7 @@ func TestCommitRequestTTL_StampDoesNotOverwriteAConcurrentExtension(t *testing.T }, }, newCommitRequest("save-now")) r := &CommitRequestReconciler{ - Client: c, APIReader: c, Finalizer: committedFinalizer(), AuthorLookup: attributedAlice(), + Client: c, APIReader: c, Finalizer: committedFinalizer(), TTL: time.Hour, } @@ -179,7 +179,7 @@ func TestCommitRequestTTL_AValueSetAtCreationIsKept(t *testing.T) { cr.Annotations = map[string]string{CommitRequestDeleteAfterAnnotation: chosen} c := newCommitRequestClient(t, nil, cr) r := &CommitRequestReconciler{ - Client: c, APIReader: c, Finalizer: committedFinalizer(), AuthorLookup: attributedAlice(), + Client: c, APIReader: c, Finalizer: committedFinalizer(), TTL: time.Hour, } diff --git a/internal/controller/constants.go b/internal/controller/constants.go index 92d636cf4..600b3c26c 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -86,18 +86,13 @@ const ( // still being established (the rule is blocked by validation), which must not be rendered as a // permanent failure. ConditionTypeSourceNamespaceAuthorized = "SourceNamespaceAuthorized" - // ConditionTypeAuthorAttributed indicates whether a CommitRequest's commit author - // was named from the submitter captured at admission. It is binary and immediately - // settled (no Unknown, no timeout): True (AttributedFromAdmission) when the - // validate-operator-types webhook recorded the submitter, False (CommitterFallback) when - // capture ran but found no record, or False (AuthorCaptureDisabled) when capture is off. - // False is not a failure and does not affect Ready: the - // request claims no actor and can attach only to an unnamed watch window - // (docs/architecture.md#commitrequest-finalize). - ConditionTypeAuthorAttributed = "AuthorAttributed" // ConditionTypePushed indicates whether a CommitRequest's commit reached the // remote repository. ConditionTypePushed = "Pushed" + // ConditionTypeApplied reports, for a CommitRequest with catchUp: AfterPush, whether + // its GitTarget's applier applied a commit that holds the request's: Unknown while it applies or + // retries, True once applied, False when there was nothing to apply. It never affects Ready. + ConditionTypeApplied = "Applied" // KubeProviderConditionValidated reports whether a KubeProvider's credential is safe and // resolvable: an inCluster provider is Validated while the installation lists it in diff --git a/internal/controller/duration_fields_admission_test.go b/internal/controller/duration_fields_admission_test.go index e7ada17b2..371000578 100644 --- a/internal/controller/duration_fields_admission_test.go +++ b/internal/controller/duration_fields_admission_test.go @@ -198,14 +198,14 @@ var _ = Describe("Duration fields", func() { omitted := &configbutleraiv1alpha3.CommitRequest{ ObjectMeta: metav1.ObjectMeta{Name: "window-omitted", Namespace: "default"}, Spec: configbutleraiv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{Name: "any-target"}, + GitTargetRef: meta.NamespacedObjectReference{Name: "any-target"}, }, } Expect(k8sClient.Create(ctx, omitted)).To(Succeed()) DeferCleanup(func() { _ = k8sClient.Delete(ctx, omitted) }) Expect(omitted.Spec.Window).NotTo(BeNil(), "the block default must be filled in") Expect(omitted.Spec.Window.Attach).To(Equal(configbutleraiv1alpha3.AttachCurrentOrNext)) - Expect(omitted.Spec.Window.AttachTimeout.Duration).To(Equal(2 * time.Second)) + Expect(omitted.Spec.Window.AttachTimeout.Duration).To(Equal(5 * time.Second)) Expect(omitted.Spec.Window.MaxDuration.Duration).To(Equal(2 * time.Second)) Expect(omitted.Spec.Window.IdleTimeout).To(BeNil(), "no idle close unless one is asked for") Expect(omitted.Spec.WhenNothingToCommit).To(Equal(configbutleraiv1alpha3.NothingToCommitResolve)) @@ -215,7 +215,7 @@ var _ = Describe("Duration fields", func() { zero := &configbutleraiv1alpha3.CommitRequest{ ObjectMeta: metav1.ObjectMeta{Name: "window-zeros", Namespace: "default"}, Spec: configbutleraiv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{Name: "any-target"}, + GitTargetRef: meta.NamespacedObjectReference{Name: "any-target"}, Window: &configbutleraiv1alpha3.CommitRequestWindow{ AttachTimeout: &metav1.Duration{}, IdleTimeout: &metav1.Duration{}, @@ -309,7 +309,7 @@ var _ = Describe("Duration fields", func() { request := &configbutleraiv1alpha3.CommitRequest{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, Spec: configbutleraiv1alpha3.CommitRequestSpec{ - GitTargetRef: meta.LocalObjectReference{Name: "any-target"}, + GitTargetRef: meta.NamespacedObjectReference{Name: "any-target"}, Message: "save", Window: &configbutleraiv1alpha3.CommitRequestWindow{ MaxDuration: &metav1.Duration{Duration: time.Minute}, diff --git a/internal/controller/gittarget_publication_test.go b/internal/controller/gittarget_publication_test.go index 2b7ab3fcc..5f2616d2f 100644 --- a/internal/controller/gittarget_publication_test.go +++ b/internal/controller/gittarget_publication_test.go @@ -96,10 +96,10 @@ func TestCommitRequestProgress_AHeldSaveSaysWhyItIsHeld(t *testing.T) { assert.Equal(t, waitingForPushMessage, plain, "a push that is merely pending says nothing more") cr := &configbutleraiv1alpha3.CommitRequest{} - markCommitRequestProgressing(cr, attributionFromAdmission, git.PhaseWaitingForPush, "") + markCommitRequestProgressing(cr, git.PhaseWaitingForPush, "") assert.True(t, progressRecorded(cr, git.PhaseWaitingForPush, "")) assert.False(t, progressRecorded(cr, git.PhaseWaitingForPush, held), "the outage is news to the save") - markCommitRequestProgressing(cr, attributionFromAdmission, git.PhaseWaitingForPush, held) + markCommitRequestProgressing(cr, git.PhaseWaitingForPush, held) assert.True(t, progressRecorded(cr, git.PhaseWaitingForPush, held)) assert.False(t, progressRecorded(cr, git.PhaseWaitingForPush, ""), "and so is the recovery") } diff --git a/internal/controller/grant_defaults_apiserver_test.go b/internal/controller/grant_defaults_apiserver_test.go index d767e1754..ed521423e 100644 --- a/internal/controller/grant_defaults_apiserver_test.go +++ b/internal/controller/grant_defaults_apiserver_test.go @@ -102,4 +102,29 @@ func TestGrantLists_APIServerDefaultsOmittedAndKeepsEmpty(t *testing.T) { require.NotNil(t, obj.Spec.AllowedApplierClaims) assert.Empty(t, obj.Spec.AllowedApplierClaims) }) + + target := func(name string, grants []configbutleraiv1alpha3.GitTargetCommitRequestGrant) *configbutleraiv1alpha3.GitTarget { + gt := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: name}, + Spec: configbutleraiv1alpha3.GitTargetSpec{Path: "apps", AllowedCommitRequests: grants}, + } + gt.Spec.GitBranchRef.Name = "main" + gt.Spec.KubeProviderRef.Name = "local" + return gt + } + + t.Run("an omitted GitTarget allowedCommitRequests reads back as the default", func(t *testing.T) { + obj := target("omitted", nil) + require.NoError(t, c.Create(ctx, obj)) + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(obj), obj)) + assert.Equal(t, configbutleraiv1alpha3.DefaultGitTargetCommitRequestGrants(), obj.Spec.AllowedCommitRequests) + }) + + t.Run("an empty GitTarget allowedCommitRequests stays empty through an update", func(t *testing.T) { + obj := target("deny-all", []configbutleraiv1alpha3.GitTargetCommitRequestGrant{}) + require.NoError(t, c.Create(ctx, obj)) + relabel(t, obj) + require.NotNil(t, obj.Spec.AllowedCommitRequests, "an update must not turn [] into the default") + assert.Empty(t, obj.Spec.AllowedCommitRequests) + }) } diff --git a/internal/controller/kubeprovider_commitrequests.go b/internal/controller/kubeprovider_commitrequests.go new file mode 100644 index 000000000..f31dd1593 --- /dev/null +++ b/internal/controller/kubeprovider_commitrequests.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/event" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// KubeProviderConditionCommitRequestsServed reports whether a KubeProvider's cluster's CommitRequests +// are served. It is absent while spec.commitRequests is Ignore, and, like AuditFactsReceived, not part +// of Ready: mirroring works without it. +const KubeProviderConditionCommitRequestsServed = "CommitRequestsServed" + +// CommitRequestsReporter is what the KubeProvider reconciler reads to report a provider's CommitRequests. +// *CommitRequestServer satisfies it. +type CommitRequestsReporter interface { + // CommitRequestsReport is the last report for provider, and false before the first one. + CommitRequestsReport(provider k8stypes.NamespacedName) (CommitRequestsReport, bool) + // CommitRequestsEvents names a provider whose report changed. + CommitRequestsEvents() <-chan event.GenericEvent +} + +// applyCommitRequestsServed writes the CommitRequestsServed condition and status.commitRequests. A +// provider that serves no CommitRequests carries neither: a condition that always said "not applicable" +// would be noise, and a True left from before the switch would be wrong. Until the server has looked at +// a provider that serves them, what is there stays. +func (r *KubeProviderReconciler) applyCommitRequestsServed( + st *reconcileStatus, provider *configbutleraiv1alpha3.KubeProvider, +) { + if !provider.ServesCommitRequests() || r.CommitRequests == nil { + st.remove(KubeProviderConditionCommitRequestsServed) + provider.Status.CommitRequests = nil + return + } + report, ok := r.CommitRequests.CommitRequestsReport(provider.Key()) + if !ok { + return + } + st.set(KubeProviderConditionCommitRequestsServed, report.Status, report.Reason, report.Message) + provider.Status.CommitRequests = &configbutleraiv1alpha3.KubeProviderCommitRequestsStatus{ + Namespaces: report.Namespaces, + } +} diff --git a/internal/controller/kubeprovider_controller.go b/internal/controller/kubeprovider_controller.go index e6b8bc0c0..9879a2f96 100644 --- a/internal/controller/kubeprovider_controller.go +++ b/internal/controller/kubeprovider_controller.go @@ -55,6 +55,10 @@ type KubeProviderReconciler struct { // absent rather than Unknown. AuditFacts AuditRouteFacts + // CommitRequests reports what each KubeProvider serves of its cluster's CommitRequests, for the + // CommitRequestsServed condition. Nil reports nothing. + CommitRequests CommitRequestsReporter + firsts kubeProviderLogFirsts // Recorder emits a Kubernetes Event on every persisted Ready transition; nil disables Events. @@ -152,6 +156,7 @@ func (r *KubeProviderReconciler) reconcileKubeProvider( // nobody posts under costs the commit AUTHOR, never the mirror, so it must not make a working // provider look broken to a kstatus reader. r.applyAuditFactsCondition(st, provider) + r.applyCommitRequestsServed(st, provider) st.applyReadiness(rd) // A failed status write is a real failure, not a verdict: propagate it so the provider is @@ -255,6 +260,12 @@ func (r *KubeProviderReconciler) SetupWithManager(mgr ctrl.Manager) error { )) } } + // A changed CommitRequest report, such as a CRD applied by hand, is written at once. + if r.CommitRequests != nil { + b = b.WatchesRawSource( + source.Channel(r.CommitRequests.CommitRequestsEvents(), &handler.EnqueueRequestForObject{}), + ) + } return b.Complete(r) } diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 5e740d588..25d5d0b07 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -182,15 +182,12 @@ var _ = BeforeSuite(func() { }).SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred()) - // A non-nil AuthorLookup that misses (no admission record) resolves to the - // committer immediately, and a Finalizer that never resolves keeps requests in the - // Progressing state. Neither ever completes, so these specs cover only the - // initial in-progress stamp and the terminal short-circuit. + // A Finalizer that never resolves keeps requests in the Progressing state, so these specs + // cover only the initial in-progress stamp and the terminal short-circuit. err = (&CommitRequestReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Finalizer: &fakeFinalizer{resolved: false}, - AuthorLookup: &fakeAuthorLookup{}, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Finalizer: &fakeFinalizer{resolved: false}, }).SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred()) diff --git a/internal/git/applier_claim.go b/internal/git/applier_claim.go index 7f4aac633..d956e2338 100644 --- a/internal/git/applier_claim.go +++ b/internal/git/applier_claim.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" itypes "github.com/ConfigButler/gitops-reverser/internal/types" @@ -242,6 +243,17 @@ type applierState struct { // named is the GitTarget incarnations that named the applier so far: one naming it for the first // time leaves a converge pending. named map[k8stypes.UID]bool + // catchUpAfter is how long the applier may stay behind the tip before a converge is pending + // for it, zero for never, as last read from its applier claim, and behindSince is when this worker first found it behind, zero while + // it is not, or since the last attempt was sent (scheduleConverges). + catchUpAfter time.Duration + behindSince time.Time + // lag is whether the converge pending is only scheduleConverges': nothing else asks for the tip, + // so its attempt yields to an edit (convergeHolds). + lag bool + // waitingSaves is the CommitRequests waiting for this applier's next apply that succeeds, each + // with the commit it pushed (save_apply.go); the attempt sent next carries them. + waitingSaves map[commitRequestID]string // appliedRevision is the revision the last operation that succeeded applied, as the applier // reported it, recovered on restart from the applier claim's status.operation. A converge of the commit // it names asks nothing. @@ -279,6 +291,25 @@ type applierState struct { followedSuspend string } +// restoreLag sets lag for the converge an attempt gives back, before it is merged: it is only +// scheduleConverges' when that is all either side holds. +func (s *applierState) restoreLag(attempt *inflightAttempt) { + switch { + case attempt.converge && !attempt.lag: + s.lag = false + case attempt.lag && !s.converge: + s.lag = true + } +} + +// onlyLag reports whether scheduleConverges' converge is all that is pending for the applier, given its +// unsent refusals: an attempt carrying nothing else yields to an edit (convergeHolds). One that carries +// a save is never only a lag, whatever lag says: a save asked for its apply, and a lag-only attempt that +// went stale would not give it back. +func (s *applierState) onlyLag(refusals map[refusalKey]unsentRefusal) bool { + return s.lag && !s.foreign && !s.adopt && !s.repair && len(refusals) == 0 && len(s.waitingSaves) == 0 +} + // repairPending reports whether a Repair is pending: not yet carried, or carried by the attempt out. func (s *applierState) repairPending() bool { return s.repair || (s.inflight != nil && s.inflight.repair) @@ -293,6 +324,11 @@ type inflightAttempt struct { converge bool repair bool refusals map[refusalKey]unsentRefusal + // lag is whether it carries only scheduleConverges' converge. + lag bool + // saves is the CommitRequests whose apply it carries, each with the commit it pushed + // (save_apply.go). + saves map[commitRequestID]string // phase is the last report of its operation, empty before the first. phase ApplyOutcome // previous is the operation recorded before this attempt's, with the attempt it belonged to, @@ -363,10 +399,12 @@ func (l *branchWorkerEventLoop) serviceAppliers() { now := time.Now() l.settleApplyResults(now) l.followRecordedOperations(now) + converging := l.scheduleConverges(now) work := l.takeReasons() // A listing that failed is retried on the applier timer (listRetryAt), which every arming below // keeps. closing := earliest(earliest(l.reportApplyWindows(now), l.snapshotHoldDeadline()), l.listRetryAt) + closing = earliest(closing, converging) if !work { l.endFlush() @@ -400,6 +438,70 @@ func (l *branchWorkerEventLoop) serviceAppliers() { l.armApplierTimer(now, wait) } +// scheduleConverges leaves a converge pending for each applier that has stayed behind the tip for its +// catchUpAfter, and returns when the next one still waiting comes due, zero when none waits. +// +// A push of the worker's own leaves nothing pending (applyInbox.observe): the flush put its edits on +// the cluster already. But the applier's own state, its applied revision and its source, stays at the +// commit before, and other things read it. The interval bounds that lag. The clock starts on the first +// pass that finds the applier behind, and starts again when an attempt is sent, so a burst of pushes +// inside one interval costs one apply, and an applier at the tip costs nothing. The converge then +// waits like any other: for the barrier, the backoff and the flush (dueAppliers), and asks nothing +// of an applier that applied the tip meanwhile. +func (l *branchWorkerEventLoop) scheduleConverges(now time.Time) time.Time { + tip := l.w.inbox.lastTip() + var next time.Time + for _, state := range l.appliers { + if tip == "" || revisionNamesCommit(state.appliedRevision, tip) { + state.behindSince = time.Time{} + continue + } + state.catchUpAfter = l.w.catchUpAfterOf(state) + if state.catchUpAfter <= 0 { + // Off: a converge only this schedule left pending, waiting on the barrier or a backoff, + // goes with it. Anything else that asks for the tip keeps it. + state.behindSince = time.Time{} + if state.lag { + state.converge, state.lag = false, false + } + continue + } + if state.behindSince.IsZero() { + state.behindSince = now + } + if state.converge || l.attemptOut(state) { + continue + } + due := state.behindSince.Add(state.catchUpAfter) + if now.Before(due) { + next = earliest(next, due) + continue + } + state.converge, state.lag = true, true + l.w.Log.V(1).Info("The applier stayed behind the tip for its catchUpAfter; a converge is pending", + "branch", l.w.Branch, "applier", state.ref.String(), "applied", state.appliedRevision, "tip", tip, + "interval", state.catchUpAfter.String()) + } + return next +} + +// catchUpAfterOf reads the applier's spec.catchUpAfter from its applier claim, on every pass +// that finds it behind, so a change applies from the next wake rather than from the next listing, +// which a push of the worker's own does not make. The client is cached, so this costs no round trip. +// An applier claim that cannot be read keeps the interval last read. +func (w *BranchWorker) catchUpAfterOf(state *applierState) time.Duration { + ref := state.ref.ApplierClaim + applierClaim := v1alpha3.NewApplierClaim(ref.Kind) + if applierClaim == nil { + return 0 + } + key := client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name} + if err := w.Client.Get(w.ctx, key, applierClaim); err != nil { + return state.catchUpAfter + } + return applierClaim.CatchUpAfter() +} + // takeReasons moves the reasons the inbox holds onto the appliers listed now, and reports whether // any applier has work. A listing that fails leaves them in the inbox, and sets listRetryAt, when // the applier timer looks again; only a listing that succeeds clears it (followRecordedOperations @@ -407,7 +509,7 @@ func (l *branchWorkerEventLoop) serviceAppliers() { func (l *branchWorkerEventLoop) takeReasons() bool { w := l.w _, foreign, converge := w.inbox.take() - if !foreign && !converge && len(l.unsentRefusals) == 0 && !l.appliersHaveWork() { + if !foreign && !converge && len(l.unsentRefusals) == 0 && len(l.savedCatchUps) == 0 && !l.appliersHaveWork() { return false } targets, err := w.appliersOnBranch(w.ctx) @@ -422,7 +524,9 @@ func (l *branchWorkerEventLoop) takeReasons() bool { for _, state := range l.appliers { state.foreign = state.foreign || foreign state.converge = state.converge || converge + state.lag = state.lag && !converge } + l.takeSavedCatchUps() return true } @@ -445,7 +549,10 @@ func (l *branchWorkerEventLoop) dueAppliers(now time.Time) ([]*applierState, tim } refusals := l.unsentRefusalsFor(state) if l.alreadyConverged(state, refusals, tip) { - state.converge = false + state.converge, state.lag = false, false + saves := state.waitingSaves + state.waitingSaves = nil + l.settleSaves(state, saves, state.appliedRevision) l.w.Log.V(1).Info("The applier already applied the tip; the converge asks nothing", "branch", l.w.Branch, "applier", state.ref.String(), "commit", tip) } @@ -620,6 +727,7 @@ func (l *branchWorkerEventLoop) syncAppliers(targets []branchTarget) { l.forgetFolders(listed) for ref, state := range l.appliers { if _, ok := named[ref]; !ok { + l.dropSaves(state) if state.inflight != nil { state.inflight.abandon() } @@ -651,7 +759,7 @@ func (l *branchWorkerEventLoop) listApplier(ref ApplierRef, named []branchTarget state.targets = named for _, target := range named { if !state.named[target.uid] { - state.named[target.uid], state.converge = true, true + state.named[target.uid], state.converge, state.lag = true, true, false } } } @@ -730,8 +838,11 @@ func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, ref id: l.nextApplyID, foreign: state.foreign, adopt: state.adopt, converge: state.converge, repair: state.repair, refusals: refusals, abandon: abandon, previous: state.operation, previousID: state.operationID, + lag: state.onlyLag(refusals), saves: state.waitingSaves, } - state.foreign, state.adopt, state.converge, state.repair = false, false, false, false + state.foreign, state.adopt, state.converge, state.repair, state.lag = false, false, false, false, false + state.waitingSaves = nil + state.behindSince = time.Time{} state.inflight = attempt state.lastTurn = attempt.id l.reportApplyWindow(state) @@ -767,12 +878,28 @@ func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, ref Record: record, Targets: targets, PriorFailures: state.failures, - Current: func() bool { return w.decisionHolds(tip) }, + Current: w.currentFor(tip, attempt.lag), Report: w.reportApplyResult, Abandoned: abandoned.Done(), }) } +// settleWindowOf ends a completed attempt's part in its applier's apply window: a window whose +// operation ran stays open for its grace, and until the watch delivered the applier's writes +// (apply_barrier.go). +func (l *branchWorkerEventLoop) settleWindowOf( + state *applierState, attempt *inflightAttempt, outcome ApplyOutcome, now time.Time, +) { + state.settleApplyWindow(attempt.id, outcome.Operated(), now) + if outcome.Operated() { + l.awaitApplyBarrier(state, now) + } + if attempt.phase == ApplyUnresolved { + l.w.Log.Info("An unresolved apply completed; its apply window closes after the grace", + "branch", l.w.Branch, "applier", state.ref.String(), "outcome", string(outcome)) + } +} + // settleApplyResults reads the reports that came back. A report of an operation still out records // where it stands, and holds the attempt. An operation that succeeded is done. Any other ending // gives back what the attempt carried and, unless it was only stale, waits out its applier's @@ -794,11 +921,7 @@ func (l *branchWorkerEventLoop) settleApplyResults(now time.Time) { state.inflight = nil attempt.abandon() l.w.raiseApplyEpoch() - state.settleApplyWindow(attempt.id, res.Outcome.Operated(), now) - if attempt.phase == ApplyUnresolved { - l.w.Log.Info("An unresolved apply completed; its apply window closes after the grace", - "branch", l.w.Branch, "applier", state.ref.String(), "outcome", string(res.Outcome)) - } + l.settleWindowOf(state, attempt, res.Outcome, now) switch res.Outcome { case ApplySucceeded: state.failures, state.retryAt = 0, time.Time{} @@ -806,8 +929,17 @@ func (l *branchWorkerEventLoop) settleApplyResults(now time.Time) { if state.appliedRevision == "" && state.operation != nil { state.appliedRevision = state.operation.Commit } + l.settleSaves(state, attempt.saves, state.appliedRevision) continue case ApplyStale: + if attempt.lag { + // It yielded to an edit, or to a newer tip: it only catches the applier's state + // up, so it waits a fresh interval rather than going again after the next flush. + state.behindSince = now + l.w.Log.V(1).Info("A converge yielded to an edit before its resume; waiting a fresh interval", + "branch", l.w.Branch, "applier", state.ref.String()) + continue + } // The decision went stale before the write: nothing failed, and the next wake // decides again, at the tip as it is then. l.restoreAttempt(state, attempt) @@ -818,6 +950,7 @@ func (l *branchWorkerEventLoop) settleApplyResults(now time.Time) { state.failures++ state.retryAt = now.Add(applierRetryDelay(state.failures, res.Outcome == ApplyRejected)) l.restoreAttempt(state, attempt) + l.retrySaves(attempt.saves, errString(res.Err)) l.lookIfSourceMoved(state, res) message := "An apply attempt did not deliver; deciding it again after its backoff" if res.Outcome == ApplyFailed { @@ -1005,8 +1138,12 @@ func (l *branchWorkerEventLoop) publishUnresolved() { func (l *branchWorkerEventLoop) restoreAttempt(state *applierState, attempt *inflightAttempt) { state.foreign = state.foreign || attempt.foreign state.adopt = state.adopt || attempt.adopt + state.restoreLag(attempt) state.converge = state.converge || attempt.converge state.repair = state.repair || attempt.repair + for id, commit := range attempt.saves { + state.waitSave(id, commit) + } for key, refusal := range attempt.refusals { if !l.w.refusalStands(key) { continue @@ -1048,6 +1185,27 @@ func (w *BranchWorker) decisionHolds(tip string) bool { return w.inbox.tipIs(tip) } +// currentFor is ApplyRequest.Current for an attempt decided at tip: decisionHolds, and for an attempt +// that only catches the applier's state up (lag), convergeHolds as well. +func (w *BranchWorker) currentFor(tip string, lag bool) func() bool { + if !lag { + return func() bool { return w.decisionHolds(tip) } + } + edits := w.liveEdits.Load() + return func() bool { return w.convergeHolds(tip, edits) } +} + +// convergeHolds is the decision of a converge that only catches the applier's state up +// (scheduleConverges): the branch is still at tip, and no live edit was mirrored since the send, the +// count of which was edits then. The fetch before the resume writes nothing to the cluster; the apply +// after it would put tip there, and an edit made meanwhile is not in tip, so the apply would revert it +// and a Repair would put it back: a flip on the cluster, for an apply nothing needed. Such a converge +// yields instead, and waits a fresh interval. An apply with any other reason does not: S1 keeps +// continuous editing from starving it, and the apply window repairs what it overwrites. +func (w *BranchWorker) convergeHolds(tip string, edits uint64) bool { + return w.decisionHolds(tip) && w.liveEdits.Load() == edits +} + // reportApplyResult is ApplyRequest.Report: it records the report and wakes the loop to settle it. // It is called from the requester's goroutine. func (w *BranchWorker) reportApplyResult(res ApplyResult) { diff --git a/internal/git/apply_barrier.go b/internal/git/apply_barrier.go new file mode 100644 index 000000000..44dbf66cb --- /dev/null +++ b/internal/git/apply_barrier.go @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "slices" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// An apply window judges a write by when its event reaches the worker, not by when the write was +// made. A grace after the operation completed covers a watch that delivers within seconds. It does +// not cover one that is behind: every event waits for its audit fact in turn, and on a source cluster +// that sends none each waits the whole attribution grace, so the applier's writes can arrive long +// after the window closed. Mirrored then, a write that reverted an edit made during the apply put the +// revert in Git and lost the edit (docs/review/20261026-apply-window-late-events.md). +// +// So a window whose operation completed also waits for the watch. For each GitTarget the applier +// covers, the watch side lists every stream's collection and takes the newest resourceVersion among +// its objects: the position of the latest write to anything in it, the applier's writes included. +// Each stream answers once it has handled an event or a bookmark at or past that position. Its answer +// rides this worker's queue, behind every event the stream routed before it, so when every stream +// has answered, every write the applier made has reached the window. The window then closes at the +// end of its grace, or at once if that has passed. +// +// A write the applier deleted is not in the list, so its event can still arrive after the window: a +// DELETE is never withheld anyway, and the object it removes is one Git no longer holds. A barrier +// that cannot be taken (the source cannot be listed, or its resourceVersions are not numbers) leaves +// the grace alone, as before. A stream that stops, or never gets that far, holds the window for at +// most applyWindowBarrierLimit. + +// applyWindowBarrierLimit bounds how long a window waits for the watch after its operation completed. +// Past it the window closes anyway, and the miss is logged and counted (barrierTimedOut). +const applyWindowBarrierLimit = 2 * time.Minute + +// ApplyBarrierFunc asks the watch side to call deliver, once per running stream of target, when that +// stream has handled every write its collection held when the barrier was taken, and returns how many +// streams will. deliver must enqueue on the worker's queue, so it lands behind every event the stream +// routed before it. It is called off the worker's loop, and may list the source cluster. +type ApplyBarrierFunc func(ctx context.Context, target itypes.ResourceReference, deliver func()) (int, error) + +// ApplyBarrier is one answer to a barrier an apply window took: a stream that got there (Passed), the +// number of streams that will answer (Registered), or the barrier failing (Err). +type ApplyBarrier struct { + Applier ApplierRef + Token uint64 + + Passed bool + Registered bool + Expected int + Err error +} + +// applyBarrierWait is what a window waits for after its operation completed. +type applyBarrierWait struct { + token uint64 + completedAt time.Time + // until bounds the wait: completedAt plus applyWindowBarrierLimit. + until time.Time + // expected is how many streams will answer, -1 until the registration arrived; passed is how many + // have. + expected int + passed int +} + +// holds reports whether the wait still keeps its window open at now. +func (b *applyBarrierWait) holds(now time.Time) bool { + return b != nil && now.Before(b.until) +} + +// Apply-window barrier outcomes, the outcome label of apply_window_barriers_total. +const ( + barrierMet = "met" + barrierTimedOut = "timed_out" + barrierFailed = "failed" +) + +// awaitApplyBarrier makes a window whose operation completed wait for the watch, when the worker has +// a barrier to take. The barrier is taken off the loop: listing a source cluster can take a while. +func (l *branchWorkerEventLoop) awaitApplyBarrier(state *applierState, now time.Time) { + window := state.window + w := l.w + if w.applyBarrier == nil || window == nil || window.closesAt.IsZero() || len(state.targets) == 0 { + return + } + l.barrierSeq++ + token := l.barrierSeq + window.barrier = &applyBarrierWait{ + token: token, completedAt: now, until: now.Add(applyWindowBarrierLimit), expected: -1, + } + ref := state.ref + targets := make([]itypes.ResourceReference, 0, len(state.targets)) + for _, target := range state.targets { + targets = append(targets, target.ref) + } + base := w.ctx + if base == nil { + base = context.Background() + } + until := window.barrier.until + go func() { + // Taking the barrier lists the source clusters; past the window's limit nothing waits for it. + ctx, cancel := context.WithDeadline(base, until) + defer cancel() + expected := 0 + for _, target := range targets { + n, err := w.applyBarrier(ctx, target, func() { + w.enqueueApplyBarrier(&ApplyBarrier{Applier: ref, Token: token, Passed: true}) + }) + if err != nil { + w.enqueueApplyBarrier(&ApplyBarrier{Applier: ref, Token: token, Err: err}) + return + } + expected += n + } + w.enqueueApplyBarrier(&ApplyBarrier{Applier: ref, Token: token, Registered: true, Expected: expected}) + }() +} + +// handleApplyBarrier counts one answer to a barrier the window waits for: its own, or one it carries +// from the window it replaced. An answer to a barrier nothing waits for any more (met, or timed out) +// is dropped. +func (l *branchWorkerEventLoop) handleApplyBarrier(answer *ApplyBarrier) { + state := l.appliers[answer.Applier] + if state == nil || state.window == nil { + return + } + window := state.window + wait := window.waitFor(answer.Token) + if wait == nil { + return + } + switch { + case answer.Err != nil: + window.dropWait(wait) + l.w.Log.Info("Could not take the apply window's barrier; the window closes after its grace alone", + "branch", l.w.Branch, "applier", state.ref.String(), "commit", window.commit, "err", answer.Err.Error()) + l.recordApplyBarrier(barrierFailed) + return + case answer.Registered: + wait.expected = answer.Expected + case answer.Passed: + wait.passed++ + } + if wait.expected < 0 || wait.passed < wait.expected { + return + } + window.dropWait(wait) + l.w.Log.V(1).Info("The watch delivered every write made before the apply completed", + "branch", l.w.Branch, "applier", state.ref.String(), "commit", window.commit, + "waited", time.Since(wait.completedAt).Round(time.Millisecond).String(), "streams", wait.expected) + l.recordApplyBarrier(barrierMet) +} + +// waitFor is the barrier with token that the window waits for, nil when none. +func (a *applyWindow) waitFor(token uint64) *applyBarrierWait { + for _, wait := range a.pendingWaits() { + if wait.token == token { + return wait + } + } + return nil +} + +// dropWait stops the window waiting for a barrier. +func (a *applyWindow) dropWait(wait *applyBarrierWait) { + if a.barrier == wait { + a.barrier = nil + return + } + a.carriedWaits = slices.DeleteFunc(a.carriedWaits, func(w *applyBarrierWait) bool { return w == wait }) +} + +// barrierTimedOut logs and counts the barriers a window still waited for when it closed. +func (l *branchWorkerEventLoop) barrierTimedOut(state *applierState) { + window := state.window + for _, wait := range window.pendingWaits() { + window.dropWait(wait) + l.w.Log.Info("The apply window closed before the watch delivered every write made before the apply "+ + "completed: a write of the applier's arriving later is mirrored as an edit", + "branch", l.w.Branch, "applier", state.ref.String(), "commit", window.commit, + "waited", time.Since(wait.completedAt).Round(time.Second).String(), + "streams", wait.expected, "delivered", wait.passed) + l.recordApplyBarrier(barrierTimedOut) + } +} + +// recordApplyBarrier counts one apply window barrier, by outcome. +func (l *branchWorkerEventLoop) recordApplyBarrier(outcome string) { + if telemetry.ApplyWindowBarriersTotal == nil { + return + } + telemetry.ApplyWindowBarriersTotal.Add(context.WithoutCancel(l.w.ctx), 1, + metric.WithAttributes(l.w.gitBranchAttrs(attribute.String("outcome", outcome))...)) +} + +// enqueueApplyBarrier puts one barrier answer on the worker's queue. It never blocks: an answer the +// full queue drops leaves its window waiting until applyWindowBarrierLimit. +func (w *BranchWorker) enqueueApplyBarrier(answer *ApplyBarrier) { + w.inflightItems.Add(1) + w.pendingResyncsMu.Lock() + defer w.pendingResyncsMu.Unlock() + if w.stoppingLocked() { + w.releaseInflightItem() + return + } + select { + case w.eventQueue <- WorkItem{ApplyBarrier: answer}: + default: + w.releaseInflightItem() + w.Log.V(1).Info("Event queue full, apply window barrier answer dropped; the window closes at its limit", + "branch", w.Branch, "applier", answer.Applier.String()) + } +} diff --git a/internal/git/apply_barrier_test.go b/internal/git/apply_barrier_test.go new file mode 100644 index 000000000..ea200eb5e --- /dev/null +++ b/internal/git/apply_barrier_test.go @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// fakeBarrier is the watch side of a barrier with one stream per GitTarget: it keeps each stream's +// deliver for the test to call, or fails. +type fakeBarrier struct { + mu sync.Mutex + delivers []func() + err error +} + +func (f *fakeBarrier) take(_ context.Context, _ itypes.ResourceReference, deliver func()) (int, error) { + if f.err != nil { + return 0, f.err + } + f.mu.Lock() + defer f.mu.Unlock() + f.delivers = append(f.delivers, deliver) + return 1, nil +} + +func (f *fakeBarrier) deliverAll() { + f.mu.Lock() + delivers := f.delivers + f.delivers = nil + f.mu.Unlock() + for _, deliver := range delivers { + deliver() + } +} + +// dispatchQueued hands every item on the worker's queue to the loop, as run() would. +func dispatchQueued(loop *branchWorkerEventLoop) { + for { + select { + case item := <-loop.w.eventQueue: + loop.dispatch(item) + loop.w.releaseInflightItem() + default: + return + } + } +} + +// completedWithBarrier is alpha's applier with an attempt that completed, its window waiting for a +// barrier whose streams have registered and not yet delivered. +func completedWithBarrier(t *testing.T, barrier *fakeBarrier) (*branchWorkerEventLoop, *applierState) { + t.Helper() + loop, req := sentAttempt(t) + loop.w.applyBarrier = barrier.take + report(req, ApplySucceeded) + loop.serviceAppliers() + state := loop.appliers[applierRef("alpha")] + require.NotNil(t, state.window) + require.Eventually(t, func() bool { + dispatchQueued(loop) + return state.window.barrier == nil || state.window.barrier.expected >= 0 + }, 5*time.Second, 10*time.Millisecond, "the barrier's registration reaches the loop") + return loop, state +} + +func barriers(t *testing.T, reader *sdkmetric.ManualReader, outcome string) int64 { + t.Helper() + got, _ := telemetry.CollectInt64Sum(reader, "gitopsreverser_apply_window_barriers_total", + map[string]string{"outcome": outcome}) + return got +} + +// A window whose operation completed stays open past its grace while the watch has not delivered +// every write made before the apply completed, so the applier's write is still judged inside it; once +// every stream delivered, it closes. +func TestApplyBarrier_HoldsTheWindowUntilEveryStreamDelivered(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + barrier := &fakeBarrier{} + loop, state := completedWithBarrier(t, barrier) + require.NotNil(t, state.window.barrier, "the window waits for the watch") + assert.Equal(t, 1, state.window.barrier.expected) + + state.window.closesAt = time.Now().Add(-time.Millisecond) + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit")), "an API edit is mirrored") + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "past its grace, the window still judges the applier's write: withheld over the edit") + + barrier.deliverAll() + dispatchQueued(loop) + assert.Nil(t, state.window.barrier, "every stream delivered") + assert.Equal(t, int64(1), barriers(t, reader, barrierMet)) + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "with its grace over and the barrier met, the window is closed") +} + +// A window waits for the watch at most applyWindowBarrierLimit. Past it, it closes, and the miss is +// counted. +func TestApplyBarrier_TheWindowClosesAtTheLimit(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, state := completedWithBarrier(t, &fakeBarrier{}) + require.NotNil(t, state.window.barrier) + + past := time.Now().Add(-time.Millisecond) + state.window.closesAt, state.window.barrier.until = past, past + loop.reportApplyWindows(time.Now()) + assert.Nil(t, state.window.barrier) + assert.Equal(t, int64(1), barriers(t, reader, barrierTimedOut)) + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), "the window is closed") +} + +// A barrier that cannot be taken leaves the window its grace alone. +func TestApplyBarrier_AFailedBarrierLeavesTheGrace(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, state := completedWithBarrier(t, &fakeBarrier{err: errors.New("the source cannot be listed")}) + + assert.Nil(t, state.window.barrier) + assert.Equal(t, int64(1), barriers(t, reader, barrierFailed)) + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), "within the grace") + state.window.closesAt = time.Now().Add(-time.Millisecond) + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), "after it") +} + +// An answer for a barrier the window no longer waits for changes nothing. +func TestApplyBarrier_AnAnswerForAnotherBarrierIsDropped(t *testing.T) { + barrier := &fakeBarrier{} + loop, state := completedWithBarrier(t, barrier) + wait := state.window.barrier + require.NotNil(t, wait) + + loop.handleApplyBarrier(&ApplyBarrier{Applier: state.ref, Token: wait.token + 1, Passed: true}) + assert.Same(t, wait, state.window.barrier) + assert.Zero(t, wait.passed) +} + +// A window opened while the one before still waits for the watch carries that wait as a wait, not as +// its limit: the earlier apply's writes are judged until its barrier is met, and once it is, the new +// window closes after its own grace. The carried barrier's answers still count after the replacement. +func TestApplyBarrier_ANewerWindowCarriesThePendingWaitNotItsLimit(t *testing.T) { + barrier := &fakeBarrier{} + loop, state := completedWithBarrier(t, barrier) + before := state.window + wait := before.barrier + require.NotNil(t, wait) + + now := time.Now() + state.openApplyWindow(before.attempt+1, "ccc", now) + next := state.window + require.Len(t, next.carriedWaits, 1, "the pending barrier is carried") + assert.Same(t, wait, next.carriedWaits[0]) + assert.True(t, next.carriedUntil.Before(wait.until), "the limit is not carried as a close time") + + state.settleApplyWindow(next.attempt, true, now) + next.closesAt = now.Add(-time.Millisecond) + assert.True(t, next.open(time.Now()), "the earlier apply's writes can still arrive") + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "the earlier apply's write is still judged inside the window") + + barrier.deliverAll() + dispatchQueued(loop) + assert.Empty(t, next.carriedWaits, "the carried barrier is met") + assert.False(t, next.open(time.Now()), "and the window closes after its own grace") +} diff --git a/internal/git/apply_converge_test.go b/internal/git/apply_converge_test.go new file mode 100644 index 000000000..edc3973cb --- /dev/null +++ b/internal/git/apply_converge_test.go @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// spec.catchUpAfter: a push of the worker's own asks nothing at once, but an applier left behind +// the tip for the interval is asked a CatchUp of it, so its own state (applied revision, source +// artifact) catches up with the branch. + +// convergedAt starts loop's applier, and has it apply the first tip, "aaa", at startup. +func convergedAt(t *testing.T, loop *branchWorkerEventLoop, requester *recordingRequester) { + t.Helper() + const commit = "aaa" + loop.w.inbox.observe(RemoteObservation{Commit: commit, By: ObservedByFetch}) + loop.serviceAppliers() + startup := requester.taken() + require.Len(t, startup, 1, "the startup converge") + startup[0].Report(ApplyResult{ID: startup[0].ID, Outcome: ApplySucceeded, AppliedRevision: commit}) + loop.serviceAppliers() + require.Empty(t, requester.taken()) +} + +// intervalPassed moves the clock of every applier's converge back by more than its interval. +func intervalPassed(t *testing.T, loop *branchWorkerEventLoop) { + t.Helper() + for _, state := range loop.appliers { + require.False(t, state.behindSince.IsZero(), "the clock runs only for an applier behind the tip") + state.behindSince = state.behindSince.Add(-state.catchUpAfter - time.Second) + } +} + +func catchUpEvery(interval time.Duration) func(*configv1alpha3.FluxClaim) { + return func(c *configv1alpha3.FluxClaim) { c.Spec.CatchUpAfter = &metav1.Duration{Duration: interval} } +} + +// TestCatchUpAfter_ABurstOfOwnPushesConvergesOnceAfterTheInterval: each push asks nothing at once and the +// timer is armed for the interval; once it has passed, one CatchUp of the newest tip goes, and an +// applier at the tip is not clocked any more. +func TestCatchUpAfter_ABurstOfOwnPushesConvergesOnceAfterTheInterval(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + convergedAt(t, loop, requester) + state := loop.appliers[applierRef("alpha")] + + for _, commit := range []string{"bbb", "ccc", "ddd"} { + loop.w.inbox.observe(RemoteObservation{Commit: commit, By: ObservedByPush}) + loop.serviceAppliers() + require.Empty(t, requester.taken(), "the worker's own push asks nothing at once") + } + require.NotNil(t, loop.applierTimer, "the timer wakes the loop when the interval has passed") + assert.Equal(t, configv1alpha3.DefaultFluxCatchUpAfter, state.catchUpAfter, "a FluxClaim's default") + + intervalPassed(t, loop) + loop.serviceAppliers() + requests := requester.taken() + require.Len(t, requests, 1, "one apply for the whole burst") + assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) + assert.Equal(t, "ddd", requests[0].Commit) + assert.True(t, state.behindSince.IsZero(), "a send starts the clock again") + + requests[0].Report(ApplyResult{ID: requests[0].ID, Outcome: ApplySucceeded, AppliedRevision: "main@sha1:ddd"}) + loop.serviceAppliers() + assert.Empty(t, requester.taken()) + assert.True(t, state.behindSince.IsZero(), "an applier at the tip is not clocked") +} + +// TestCatchUpAfter_APushDuringAnAttemptWaitsAFreshInterval: the attempt applied the tip it was sent for, +// and the push that landed meanwhile waits an interval from the send, not from before it. +func TestCatchUpAfter_APushDuringAnAttemptWaitsAFreshInterval(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + convergedAt(t, loop, requester) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + loop.serviceAppliers() + intervalPassed(t, loop) + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) + loop.serviceAppliers() + sent[0].Report(ApplyResult{ID: sent[0].ID, Outcome: ApplySucceeded, AppliedRevision: "bbb"}) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "behind again, but the interval starts at the send") + + intervalPassed(t, loop) + loop.serviceAppliers() + requests := requester.taken() + require.Len(t, requests, 1) + assert.Equal(t, "ccc", requests[0].Commit) +} + +// TestCatchUpAfter_ZeroNeverConverges: "0s" keeps a push of the worker's own asking nothing, ever; an +// ArgoCDClaim defaults to it, because a sync runs the Application's hooks. +func TestCatchUpAfter_ZeroNeverConverges(t *testing.T) { + for name, objects := range map[string][]client.Object{ + "flux 0s": {applierTestTarget("alpha", true), fluxClaim("alpha", catchUpEvery(0))}, + "argo cd": {applierTestTarget("alpha", true, argoApplier)}, + } { + t.Run(name, func(t *testing.T) { + loop, requester := manualLoop(t, objects...) + convergedAt(t, loop, requester) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + loop.serviceAppliers() + + for _, state := range loop.appliers { + assert.Zero(t, state.catchUpAfter) + assert.True(t, state.behindSince.IsZero(), "no clock runs") + } + assert.Empty(t, requester.taken()) + }) + } +} + +// TestCatchUpAfter_YieldsToAnEditBeforeItsResume: an interval converge's decision stops holding once a +// live edit is mirrored after its send, since the tip it fetched lacks that edit and applying it would +// flip the object on the cluster. Stale, it waits a fresh interval, then goes for the newer tip. +func TestCatchUpAfter_YieldsToAnEditBeforeItsResume(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + convergedAt(t, loop, requester) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + loop.serviceAppliers() + intervalPassed(t, loop) + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + require.True(t, sent[0].Current(), "nothing happened since the send") + + loop.handleLiveEvents(&WriteRequest{Events: []Event{windowEvent("alpha", "2", "kubectl-edit")}}) + assert.False(t, sent[0].Current(), "an edit arrived during the fetch") + + report(sent[0], ApplyStale) + loop.openWindow = nil + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "not again at once: a fresh interval") + state := loop.appliers[applierRef("alpha")] + assert.False(t, state.converge) + + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) + intervalPassed(t, loop) + loop.serviceAppliers() + again := requester.taken() + require.Len(t, again, 1) + assert.Equal(t, "ccc", again[0].Commit) +} + +// TestCatchUpAfter_AnyOtherReasonDoesNotYield: a ForeignCommit is not a catch-up, so an edit does not make +// it stale; the apply window repairs what it overwrites (S1). +func TestCatchUpAfter_AnyOtherReasonDoesNotYield(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + convergedAt(t, loop, requester) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + require.Equal(t, configv1alpha3.ApplyReasonForeignCommit, sent[0].Reason) + + loop.handleLiveEvents(&WriteRequest{Events: []Event{windowEvent("alpha", "2", "kubectl-edit")}}) + assert.True(t, sent[0].Current()) +} + +// TestCatchUpAfter_AChangedIntervalAppliesFromTheNextWake: the interval is read from the applier claim on +// every pass that finds the applier behind, so a change needs no listing, which a push of the +// worker's own does not make. +func TestCatchUpAfter_AChangedIntervalAppliesFromTheNextWake(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true), fluxClaim("alpha", catchUpEvery(0))) + convergedAt(t, loop, requester) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + loop.serviceAppliers() + state := loop.appliers[applierRef("alpha")] + require.True(t, state.behindSince.IsZero(), "off") + + claim := &configv1alpha3.FluxClaim{} + require.NoError(t, loop.w.Client.Get(t.Context(), client.ObjectKey{Namespace: "team-a", Name: "alpha"}, claim)) + catchUpEvery(30 * time.Second)(claim) + require.NoError(t, loop.w.Client.Update(t.Context(), claim)) + + loop.serviceAppliers() + assert.Equal(t, 30*time.Second, state.catchUpAfter) + intervalPassed(t, loop) + loop.serviceAppliers() + requests := requester.taken() + require.Len(t, requests, 1) + assert.Equal(t, "bbb", requests[0].Commit) +} + +// TestCatchUpAfter_ASaveThatAsksCatchesUpAtOnce: a CommitRequest with catchUp: AfterPush +// leaves a converge pending for its GitTarget's applier once its commit is pushed, without waiting +// for catchUpAfter; one that pushed no commit asks nothing. It does not yield to an edit: it was +// asked for. +func TestCatchUpAfter_ASaveThatAsksCatchesUpAtOnce(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + convergedAt(t, loop, requester) + loop.pendingCRs = map[commitRequestID]*pendingCommitRequest{} + save := func(name string, commit string) { + id := commitRequestID{Namespace: "team-a", Name: name, UID: "uid-" + name} + loop.pendingCRs[id] = &pendingCommitRequest{ + id: id, gitTargetName: "alpha", gitTargetNamespace: "team-a", catchUpNow: true, + } + loop.resolveCommitRequest(id, FinalizeResult{Outcome: FinalizeCommitted, Commit: commit}) + } + + save("nothing-pushed", "") + loop.serviceAppliers() + require.Empty(t, requester.taken(), "no commit, nothing to catch up with") + + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + save("pushed", "bbb") + loop.serviceAppliers() + requests := requester.taken() + require.Len(t, requests, 1, "at once, not after catchUpAfter") + assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) + assert.Equal(t, "bbb", requests[0].Commit) + + loop.handleLiveEvents(&WriteRequest{Events: []Event{windowEvent("alpha", "2", "kubectl-edit")}}) + assert.True(t, requests[0].Current(), "asked for, so it does not yield to an edit") +} + +// TestSaveApply_FollowsTheAttemptThatCarriesIt: a save that asked to catch up waits on its applier, +// is carried by the attempt sent next, is retried when that attempt does not succeed, and is +// settled applied by the one that does. One that pushed nothing, or whose GitTarget names no applier, +// is settled at once as applied: no apply is needed. +func TestSaveApply_FollowsTheAttemptThatCarriesIt(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true), applierTestTarget("loner", false)) + convergedAt(t, loop, requester) + loop.pendingCRs = map[commitRequestID]*pendingCommitRequest{} + save := func(name, target, commit string) commitRequestID { + id := commitRequestID{Namespace: "team-a", Name: name, UID: "uid-" + name} + loop.pendingCRs[id] = &pendingCommitRequest{ + id: id, gitTargetName: target, gitTargetNamespace: "team-a", catchUpNow: true, + } + loop.resolveCommitRequest(id, FinalizeResult{Outcome: FinalizeCommitted, Commit: commit}) + return id + } + lookup := func(id commitRequestID) SaveApply { + apply, known := loop.w.LookupSaveApply(id.Namespace, id.Name, id.UID) + require.True(t, known, "the worker knows %s", id.Name) + return apply + } + + nothing := save("nothing", "alpha", "") + assert.Equal(t, SaveApply{Done: true, Applied: true, Reason: SaveApplyNothingToApply, + Message: lookup(nothing).Message}, lookup(nothing)) + + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + pushed := save("pushed", "alpha", "bbb") + alone := save("alone", "loner", "bbb") + assert.Equal(t, SaveApplyApplying, lookup(pushed).Reason) + loop.serviceAppliers() + assert.True(t, lookup(alone).Done) + assert.True(t, lookup(alone).Applied, "no applier applies the branch; the cluster holds the save") + assert.Equal(t, SaveApplyNoApplier, lookup(alone).Reason) + + first := requester.taken() + require.Len(t, first, 1) + report(first[0], ApplyFailed) + loop.serviceAppliers() + assert.Equal(t, SaveApplyRetrying, lookup(pushed).Reason) + assert.False(t, lookup(pushed).Done) + + dueNow(loop) + loop.serviceAppliers() + again := requester.taken() + require.Len(t, again, 1, "the retry carries the save") + again[0].Report(ApplyResult{ID: again[0].ID, Outcome: ApplySucceeded, AppliedRevision: "main@sha1:bbb"}) + loop.serviceAppliers() + settled := lookup(pushed) + assert.True(t, settled.Done) + assert.True(t, settled.Applied) + assert.Equal(t, "main@sha1:bbb", settled.AppliedRevision) +} + +// TestSaveApply_AnApplierAlreadyAtTheTipSettlesIt: when the applier applied the tip before the save's +// converge went, nothing is asked, and the save is applied all the same. +func TestSaveApply_AnApplierAlreadyAtTheTipSettlesIt(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + convergedAt(t, loop, requester) + loop.pendingCRs = map[commitRequestID]*pendingCommitRequest{} + id := commitRequestID{Namespace: "team-a", Name: "late", UID: "uid-late"} + loop.pendingCRs[id] = &pendingCommitRequest{ + id: id, + gitTargetName: "alpha", + gitTargetNamespace: "team-a", + catchUpNow: true, + } + loop.resolveCommitRequest(id, FinalizeResult{Outcome: FinalizeCommitted, Commit: "aaa"}) + + loop.serviceAppliers() + assert.Empty(t, requester.taken()) + apply, known := loop.w.LookupSaveApply(id.Namespace, id.Name, id.UID) + require.True(t, known) + assert.True(t, apply.Applied) +} + +// TestCatchUpAfter_TurningItOffCancelsAPendingConverge: a converge only catchUpAfter left pending, +// waiting out the backoff of an attempt that failed, goes when catchUpAfter becomes "0s". A save that +// asks to catch up still goes: catchUpAfter does not decide it. +func TestCatchUpAfter_TurningItOffCancelsAPendingConverge(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true), fluxClaim("alpha", catchUpEvery(30*time.Second))) + convergedAt(t, loop, requester) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + loop.serviceAppliers() + intervalPassed(t, loop) + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + report(sent[0], ApplyFailed) + loop.serviceAppliers() + state := loop.appliers[applierRef("alpha")] + require.True(t, state.converge, "given back, waiting out its backoff") + + claim := &configv1alpha3.FluxClaim{} + require.NoError(t, loop.w.Client.Get(t.Context(), client.ObjectKey{Namespace: "team-a", Name: "alpha"}, claim)) + catchUpEvery(0)(claim) + require.NoError(t, loop.w.Client.Update(t.Context(), claim)) + dueNow(loop) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "off: the timed converge is gone") + assert.False(t, state.converge) + + loop.pendingCRs = map[commitRequestID]*pendingCommitRequest{} + id := commitRequestID{Namespace: "team-a", Name: "asks", UID: "uid-asks"} + loop.pendingCRs[id] = &pendingCommitRequest{ + id: id, gitTargetName: "alpha", gitTargetNamespace: "team-a", catchUpNow: true, + } + loop.resolveCommitRequest(id, FinalizeResult{Outcome: FinalizeCommitted, Commit: "bbb"}) + loop.serviceAppliers() + asked := requester.taken() + require.Len(t, asked, 1, "a save that asks is no timed converge") + assert.Equal(t, "bbb", asked[0].Commit) +} + +// saveHistory makes the worker's checkout hold base, saved on it, later on saved, and rewritten on base +// beside saved: the history a force-push that drops saved leaves. It returns them in that order. +func saveHistory(t *testing.T, w *BranchWorker) (string, string, string, string) { + t.Helper() + w.GitBranchUID = k8stypes.UID("uid-" + strings.ToLower(t.Name())) + dir := w.repoPath() + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + repo, err := gogit.PlainInit(dir, false) + require.NoError(t, err) + require.NoError(t, PinExplicitSigningPolicy(repo)) + worktree, err := repo.Worktree() + require.NoError(t, err) + // In the folder of applierTestTarget("alpha"), which an apply waits for. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "apps", "alpha"), 0o750)) + file := filepath.Join("apps", "alpha", "a.yaml") + baseHash := commitFileChange(t, worktree, dir, file, "base") + savedHash := commitFileChange(t, worktree, dir, file, "saved") + laterHash := commitFileChange(t, worktree, dir, file, "later") + require.NoError(t, worktree.Checkout(&gogit.CheckoutOptions{Hash: baseHash, Force: true})) + rewrittenHash := commitFileChange(t, worktree, dir, file, "rewritten") + return baseHash.String(), savedHash.String(), laterHash.String(), rewrittenHash.String() +} + +// TestSaveApply_AppliedOnlyWhenTheRevisionHoldsItsCommit: an apply that succeeds settles a save as +// applied only when the applied revision holds the save's commit. One whose commit a force-push +// dropped from the branch is settled as not applied; one the checkout cannot tell about waits. +func TestSaveApply_AppliedOnlyWhenTheRevisionHoldsItsCommit(t *testing.T) { + cases := map[string]struct { + tip func(base, saved, later, rewritten string) (string, ObservationSource) + applied func(tip string) string + want SaveApply + }{ + "a later commit holds it": { + tip: func(_, _, later, _ string) (string, ObservationSource) { return later, ObservedByPush }, + applied: func(tip string) string { return "main@sha1:" + tip }, + want: SaveApply{Done: true, Applied: true, Reason: SaveApplyApplied}, + }, + "a force-push dropped it": { + tip: func(_, _, _, rewritten string) (string, ObservationSource) { return rewritten, ObservedByFetch }, + applied: func(tip string) string { return "main@sha1:" + tip }, + want: SaveApply{Done: true, Reason: SaveApplyCommitDropped}, + }, + "the checkout lacks the applied commit": { + tip: func(_, _, later, _ string) (string, ObservationSource) { return later, ObservedByPush }, + applied: func(string) string { return "main@sha1:" + strings.Repeat("f", 40) }, + want: SaveApply{Reason: SaveApplyRetrying}, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + base, saved, later, rewritten := saveHistory(t, loop.w) + loop.w.inbox.observe(RemoteObservation{Commit: base, By: ObservedByFetch}) + loop.serviceAppliers() + startup := requester.taken() + require.Len(t, startup, 1) + startup[0].Report(ApplyResult{ID: startup[0].ID, Outcome: ApplySucceeded, AppliedRevision: base}) + loop.serviceAppliers() + + loop.pendingCRs = map[commitRequestID]*pendingCommitRequest{} + id := commitRequestID{Namespace: "team-a", Name: "save", UID: "uid-save"} + loop.pendingCRs[id] = &pendingCommitRequest{ + id: id, gitTargetName: "alpha", gitTargetNamespace: "team-a", catchUpNow: true, + } + loop.resolveCommitRequest(id, FinalizeResult{Outcome: FinalizeCommitted, Commit: saved}) + tip, by := tc.tip(base, saved, later, rewritten) + loop.w.inbox.observe(RemoteObservation{Commit: tip, By: by}) + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + require.Equal(t, tip, sent[0].Commit) + sent[0].Report(ApplyResult{ID: sent[0].ID, Outcome: ApplySucceeded, AppliedRevision: tc.applied(tip)}) + loop.serviceAppliers() + + got, known := loop.w.LookupSaveApply(id.Namespace, id.Name, id.UID) + require.True(t, known) + assert.Equal(t, tc.want.Done, got.Done, got.Message) + assert.Equal(t, tc.want.Applied, got.Applied, got.Message) + assert.Equal(t, tc.want.Reason, got.Reason, got.Message) + if !got.Done { + assert.Contains(t, loop.appliers[applierRef("alpha")].waitingSaves, id, "it waits on") + } + }) + } +} + +// An attempt that carries a save is never a lag-only converge, so it never yields to an edit and a +// stale one gives its saves back. +func TestApplierState_ASaveIsNeverOnlyALag(t *testing.T) { + state := &applierState{lag: true} + assert.True(t, state.onlyLag(nil)) + state.waitSave(commitRequestID{Namespace: "team-a", Name: "save", UID: "uid"}, "ccc") + assert.False(t, state.onlyLag(nil)) +} diff --git a/internal/git/apply_window.go b/internal/git/apply_window.go index 565d36da3..dba110fa8 100644 --- a/internal/git/apply_window.go +++ b/internal/git/apply_window.go @@ -126,31 +126,76 @@ type applyWindow struct { // time of the latest such write's managedFields entries (zero when the event carried none). edited map[string]time.Time // carried holds the objects the window before this one had seen edited, when that window was - // still open as this attempt was sent, and carriedUntil is when it would have closed: the - // earlier apply's writes can arrive until then (openApplyWindow). carriedSentAt is the earliest - // send of the windows carried, zero when none is. + // still open as this attempt was sent, and carriedUntil is when its grace would have ended; + // carriedWaits are its barriers still waiting for the watch. The earlier apply's writes can arrive + // until both have passed (openApplyWindow). carriedSentAt is the earliest send of the windows + // carried, zero when none is. carried map[string]time.Time carriedUntil time.Time + carriedWaits []*applyBarrierWait carriedSentAt time.Time // echoes counts the applier's writes the window kept out of Git: the apply landing, object by // object. reported is set once that count was logged (reportApplyWindows). echoes int reported bool + // barrier is what the window waits for once its operation completed: the watch delivering every + // write the GitTargets' source clusters held by then (apply_barrier.go). Nil when it waits for + // none, or once it was met. + barrier *applyBarrierWait } // earliestSend is the lower bound of the applier's own writes at now: the earliest send of this // window and of the windows it still carries. A delayed echo of the attempt before, arriving after // this one was sent and inside that one's grace, is still that apply. func (a *applyWindow) earliestSend(now time.Time) time.Time { - if !a.carriedSentAt.IsZero() && now.Before(a.carriedUntil) { + if !a.carriedSentAt.IsZero() && a.carrying(now) { return a.carriedSentAt } return a.sentAt } -// open reports whether the window is open at now. +// carrying reports whether the windows this one carries could still see their applies' writes arrive +// at now: until their grace has passed, and while one of their barriers still waits for the watch. +func (a *applyWindow) carrying(now time.Time) bool { + return now.Before(a.carriedUntil) || a.carriedWaitHolds(now) +} + +// carriedWaitHolds reports whether a barrier carried from the window before still waits at now. +func (a *applyWindow) carriedWaitHolds(now time.Time) bool { + return slices.ContainsFunc(a.carriedWaits, func(wait *applyBarrierWait) bool { return wait.holds(now) }) +} + +// open reports whether the window is open at now: until its grace has passed, which settling the +// window extends to the grace it carries, and for as long as it, or a window it carries, still waits +// for the watch (barrier). func (a *applyWindow) open(now time.Time) bool { - return a != nil && (a.closesAt.IsZero() || now.Before(a.closesAt)) + return a != nil && + (a.closesAt.IsZero() || now.Before(a.closesAt) || a.barrier.holds(now) || a.carriedWaitHolds(now)) +} + +// closesBy is when the window closes at the latest: zero while the attempt is out, and past the grace +// while a barrier waits for the watch. +func (a *applyWindow) closesBy() time.Time { + if a.closesAt.IsZero() { + return a.closesAt + } + by := latest(a.closesAt, a.carriedUntil) + if a.barrier != nil { + by = latest(by, a.barrier.until) + } + for _, wait := range a.carriedWaits { + by = latest(by, wait.until) + } + return by +} + +// pendingWaits are the barriers that hold the window: its own and the ones it carries. +func (a *applyWindow) pendingWaits() []*applyBarrierWait { + waits := slices.Clone(a.carriedWaits) + if a.barrier != nil { + waits = append(waits, a.barrier) + } + return waits } // objectKey is an event's object without its API version: a preferred-version bump during the @@ -166,7 +211,7 @@ func objectKey(id itypes.ResourceIdentifier) string { // attempt's apply of a commit that holds every edit published before it (S1). func (a *applyWindow) editedSince(key string, now, written time.Time) (time.Time, bool) { at, edited := a.edited[key] - if carriedAt, carried := a.carried[key]; carried && now.Before(a.carriedUntil) && + if carriedAt, carried := a.carried[key]; carried && a.carrying(now) && written.Before(a.sentAt.Truncate(time.Second)) { return latest(at, carriedAt), true } @@ -179,18 +224,24 @@ func (a *applyWindow) recordEdit(key string, at time.Time) { } // openApplyWindow starts an applier's window as its attempt is sent. The window before it can still -// be open: an attempt may follow one that completed within its grace, and that apply's writes are -// still arriving. Its edits are carried until it would have closed, so a late write of the earlier -// apply over one of them is still withheld. Replacing the window lost them, and the revert was -// mirrored. +// be open: an attempt may follow one that completed within its grace, or whose barrier still waits for +// the watch, and that apply's writes are still arriving. Its edits are carried until its grace ends +// and its barriers are met, so a late write of the earlier apply over one of them is still withheld. +// Replacing the window lost them, and the revert was mirrored. Its barriers are carried as they are, +// not as their limits: one that is met stops holding the new window. func (state *applierState) openApplyWindow(attempt uint64, commit string, now time.Time) { next := &applyWindow{attempt: attempt, commit: commit, sentAt: now, edited: map[string]time.Time{}} if before := state.window; before.open(now) { next.carried = maps.Clone(before.edited) - if now.Before(before.carriedUntil) { + if before.carrying(now) { maps.Copy(next.carried, before.carried) } - next.carriedUntil = before.closesAt + next.carriedUntil = latest(before.closesAt, before.carriedUntil) + for _, wait := range before.pendingWaits() { + if wait.holds(now) { + next.carriedWaits = append(next.carriedWaits, wait) + } + } next.carriedSentAt = before.earliestSend(now) } state.window = next @@ -207,7 +258,7 @@ func (state *applierState) settleApplyWindow(attempt uint64, operated bool, now switch { case operated: window.closesAt = latest(now.Add(applyWindowGrace), window.carriedUntil) - case now.Before(window.carriedUntil): + case window.carrying(now): window.closesAt = window.carriedUntil default: state.window = nil @@ -341,9 +392,10 @@ func (l *branchWorkerEventLoop) reportApplyWindows(now time.Time) time.Time { switch { case window == nil || window.reported: case !window.open(now): + l.barrierTimedOut(state) l.reportApplyWindow(state) - case !window.closesAt.IsZero() && (next.IsZero() || window.closesAt.Before(next)): - next = window.closesAt + case !window.closesAt.IsZero() && (next.IsZero() || window.closesBy().Before(next)): + next = window.closesBy() } } return next diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index e8f0ea65d..ba4775257 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -163,6 +163,10 @@ type BranchWorker struct { applyEpoch atomic.Uint64 appliersAhead atomic.Bool hasAppliers atomic.Bool + // liveEdits counts the live events the loop has mirrored, for the requester's goroutine to read: a + // converge that only catches the applier's state up yields to an edit that arrives before its + // resume (convergeHolds). + liveEdits atomic.Uint64 // initialSnapshots reports whether every collection a GitTarget watches has a settled // snapshot, and resyncRecheck asks the watch side for fresh snapshots of a GitTarget. Set by the // WorkerManager before Start; nil in the CLI and in tests (apply_epoch.go). @@ -172,6 +176,10 @@ type BranchWorker struct { // admits every GitTarget. applierAdmitted ApplierAdmission resyncRecheck func(target itypes.ResourceReference) + // applyBarrier lets an apply window wait for the watch after its operation completed + // (apply_barrier.go). Set by the WorkerManager; nil in the CLI and in tests, where a window closes + // after its grace alone. + applyBarrier ApplyBarrierFunc // replacementPending reports whether this worker replaced one whose GitTargets have not been told // yet (WorkerManager.ReplacementPending): until they are, their snapshot record is the old // worker's, and no applier is asked for anything. Nil in the CLI and in tests. @@ -366,6 +374,9 @@ type BranchWorker struct { // crPhases holds where each unresolved CommitRequest stands, for the controller to report // while it waits. Guarded by crOutcomesMu; the event loop is the only writer. crPhases map[commitRequestID]CommitRequestPhase + // crApplies holds where the apply each CommitRequest asked for stands (save_apply.go). Guarded by + // crOutcomesMu; the event loop is the only writer. + crApplies map[commitRequestID]saveApplyEntry // crQueuedAttaches counts, per request, the attaches on the FIFO that the loop has not handled // yet. An outcome is not collected while any are queued: the outcome is what turns a late attach // into a no-op, so collecting it first would let that attach register a withdrawn request @@ -999,6 +1010,9 @@ func (w *BranchWorker) processEvents() { type branchWorkerEventLoop struct { w *BranchWorker + // barrierSeq numbers the barriers this loop's apply windows take (apply_barrier.go). + barrierSeq uint64 + // defaultWindow is the operator-level default for a GitTarget's commit window timers, used // for any target that declares no spec.commit.window field and for one that cannot be read. defaultWindow commitWindowDefaults @@ -1040,6 +1054,10 @@ type branchWorkerEventLoop struct { // window opens (then it attaches to it) or its attach deadline passes. // Loop-goroutine only. pendingCRs map[commitRequestID]*pendingCommitRequest + // savedCatchUps is the CommitRequests, by GitTarget, whose pushed commit asked their applier to + // catch up at once (catchUp: AfterPush), with that commit, until the appliers are next listed + // (takeReasons, save_apply.go). + savedCatchUps map[itypes.ResourceReference]map[commitRequestID]string // crSeq numbers registrations, so competing requests attach first come, first served. crSeq uint64 // refusalTimer fires at the earliest deadline in refusalPending. See refusal_reapply.go. @@ -1244,6 +1262,8 @@ func (l *branchWorkerEventLoop) dispatch(item WorkItem) int64 { l.handleAttachCommitRequest(item.Attach) case item.Withdraw != nil: l.handleWithdrawCommitRequest(item.Withdraw) + case item.ApplyBarrier != nil: + l.handleApplyBarrier(item.ApplyBarrier) case item.Resync != nil: // Take the CURRENT request for this scope: a newer one may have replaced it while this // marker waited in the FIFO. Running it here keeps the original queue position, so a resync @@ -1275,6 +1295,7 @@ func (l *branchWorkerEventLoop) handleLiveEvents(request *WriteRequest) { if !l.mirrors(event) { continue } + l.w.liveEdits.Add(1) l.appendLiveEvent(event) } } diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index 2a0ff5d85..4b05f606d 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -53,6 +53,11 @@ const ( // produced no diff — the change already matches the remote, so no commit was // made (loop prevention). Resolved at finalize, never waiting on a push. FinalizeAlreadyPresent FinalizeOutcome = "AlreadyPresent" + // FinalizeSaveInProgress means an earlier request of the same author on the same GitTarget was + // still waiting for a window, or collecting the open one, when this one was registered. Both + // would race for one window, which carries one message, so this one is refused at once rather + // than left to lose its message to the first, or its edits to the first's commit. + FinalizeSaveInProgress FinalizeOutcome = "SaveInProgress" ) // CommitRequestPhase is where an unresolved CommitRequest stands on the worker. It is what the @@ -131,6 +136,10 @@ type AttachCommitRequest struct { // GitTargetName / GitTargetNamespace scope the finalize to one GitTarget. GitTargetName string GitTargetNamespace string + // GitTargetUID is the UID of the GitTarget that admitted the request. A GitTarget of that name + // with another UID replaced it and admitted nothing, so it is never asked to take the request. + // Empty matches any. + GitTargetUID string // Message is the verbatim commit message to attach to the window. Empty keeps // the generated grouped-commit message. @@ -146,6 +155,9 @@ type AttachCommitRequest struct { // CommitEmpty records Message in an empty commit when the request ends with nothing to // commit: whenNothingToCommit: CommitEmpty. CommitEmpty bool + // CatchUpNow asks the GitTarget's applier to apply the tip as soon as the request's commit is + // pushed: catchUp: AfterPush. + CatchUpNow bool } // commitRequestID is the worker-local key for a CommitRequest: its namespaced @@ -184,6 +196,8 @@ type pendingCommitRequest struct { // commitEmpty records the message in an empty commit when the request ends with nothing to // commit. Never for a WindowMismatch: that would claim more than happened. commitEmpty bool + // catchUpNow asks the GitTarget's applier for a converge once the request's commit is pushed. + catchUpNow bool // attached is true once this request is bound to a write: its message to the open window, or // its own empty-commit record. From then on only that write's outcome settles it, and it stays // set while the write is pending, waiting for the push. @@ -246,10 +260,17 @@ func (p *pendingCommitRequest) matchesWindow(w *openWindow) bool { if p == nil || w == nil { return false } - return p.author == w.Author && - p.attribution.NamesActor() == w.Attribution.NamesActor() && - p.gitTargetName == w.GitTarget && - p.gitTargetNamespace == w.GitTargetNamespace + if p.gitTargetName != w.GitTarget || p.gitTargetNamespace != w.GitTargetNamespace { + return false + } + return p.author == w.Author && p.attribution.NamesActor() == w.Attribution.NamesActor() +} + +// sameSaver reports whether two requests would save one window: the same GitTarget and the same +// author, by the comparison matchesWindow makes. +func (p *pendingCommitRequest) sameSaver(other *pendingCommitRequest) bool { + return p.gitTargetName == other.gitTargetName && p.gitTargetNamespace == other.gitTargetNamespace && + p.author == other.author && p.attribution.NamesActor() == other.attribution.NamesActor() } // commitRequestOutcomeEntry is a resolved outcome retained for the controller to diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index ee3123151..9800534bb 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -60,6 +60,7 @@ func (w *BranchWorker) recordCommitRequestOutcomeLocked(id commitRequestID, resu w.crOwners.release(k, w) } } + w.collectSaveApplies(now) } // countQueuedAttach records an attach entering the FIFO. The caller holds pendingResyncsMu, so the @@ -213,6 +214,14 @@ func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitReque idleTimeout: req.IdleTimeout, maxDuration: req.MaxDuration, commitEmpty: req.CommitEmpty, + catchUpNow: req.CatchUpNow, + } + if earlier := l.saveInProgress(pcr, req.Attach); earlier != nil { + l.w.recordCommitRequestOutcome(id, FinalizeResult{Branch: l.w.Branch, Outcome: FinalizeSaveInProgress}) + l.w.Log.Info("CommitRequest refused: an earlier save of the same author on the GitTarget is in progress", + "request", id.Namespace+"/"+id.Name, "earlier", earlier.id.Namespace+"/"+earlier.id.Name, + "author", req.Author, "target", req.GitTargetNamespace+"/"+req.GitTargetName) + return } l.pendingCRs[id] = pcr l.w.setCommitRequestPhase(id, PhaseWaitingForWindow) @@ -248,6 +257,31 @@ func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitReque } } +// saveInProgress returns the earlier request a newly registered one would race for one window, or nil. +// That is one of the same author on the same GitTarget (sameSaver) that is still waiting for a window, +// or that is collecting the open one. A request that asks for the Next window does not race one that +// is collecting: it closes that window under the earlier request's message first. A request already +// waiting for its push holds no window, so it races nothing. +func (l *branchWorkerEventLoop) saveInProgress( + pcr *pendingCommitRequest, attach v1alpha3.AttachPolicy, +) *pendingCommitRequest { + now := time.Now() + for _, earlier := range l.pendingCRs { + if !earlier.sameSaver(pcr) { + continue + } + if earlier.waiting(now) { + return earlier + } + collecting := earlier.attached && l.openWindow != nil && l.openWindow.pendingCR != nil && + *l.openWindow.pendingCR == earlier.id + if collecting && attach != v1alpha3.AttachNext { + return earlier + } + } + return nil +} + // serviceCommitRequests runs after every loop wake: note foreign windows, resolve waiting requests // whose attach deadline passed, attach the next waiting request to an open window, and re-arm the // attach timer. @@ -472,6 +506,9 @@ func (l *branchWorkerEventLoop) resolveCommitRequest(id commitRequestID, result result.Branch = l.w.Branch } l.w.recordCommitRequestOutcome(id, result) + if pcr := l.pendingCRs[id]; pcr != nil && pcr.catchUpNow { + l.catchUpAfterSave(pcr, result.Commit) + } delete(l.pendingCRs, id) l.w.setCommitRequestPhase(id, "") l.w.Log.Info("CommitRequest resolved", diff --git a/internal/git/commit_request_attach_test.go b/internal/git/commit_request_attach_test.go index dabbc2e0d..b68e7473f 100644 --- a/internal/git/commit_request_attach_test.go +++ b/internal/git/commit_request_attach_test.go @@ -428,36 +428,91 @@ func TestAttach_ForeignWindowClosingBeforeExpiryIsStillAMismatch(t *testing.T) { "bob was refused by a window that has since closed; the refusal still happened") } -// A window bob COULD have claimed, already taken by another of his own saves, is contention rather -// than a mismatch: it resolves on the next window, and calling it a mismatch would blame a -// queueing delay on the wrong cause. -func TestAttach_ClaimedSameAuthorWindowIsNotAMismatch(t *testing.T) { - worker, _, _ := setupCommitPushSplitWorker(t) - createPlainGitTarget(t, worker, "team-a", "team-a") +// A second save of one author on one GitTarget, made while the first still waits for a window or +// collects one, would race it for that window, which carries one message: the second is refused at +// once, SaveInProgress, instead of losing its message or its edits to the first. It is not a +// mismatch, and it holds nothing. +func TestAttach_ASecondSaveOfTheSameAuthorIsRefusedWhileTheFirstIsInProgress(t *testing.T) { + second := func() *AttachCommitRequest { + req := attachReq("alice", 60*time.Second) + req.Name, req.UID = "save-second", "uid-save-second" + return req + } + secondOutcome := func(t *testing.T, w *BranchWorker) (FinalizeResult, bool) { + t.Helper() + return w.LookupCommitRequestOutcome("default", "save-second", "uid-save-second") + } - loop := newBranchWorkerEventLoop(worker, time.Hour) - loop.lastPushAt = time.Now() - defer loop.stopTimers() + t.Run("while the first collects the window", func(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + loop.lastPushAt = time.Now() + defer loop.stopTimers() + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, + }}) + serviceAttach(loop, attachReq("alice", 60*time.Second)) + require.NotNil(t, loop.openWindow.pendingCR, "precondition: the first save collects the window") - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, - }}) - require.NotNil(t, loop.openWindow) + serviceAttach(loop, second()) - // Alice's first save claims the window; her second one has to wait for the next. - first := attachReq("alice", 60*time.Second) - serviceAttach(loop, first) - require.NotNil(t, loop.openWindow.pendingCR, "precondition: the window is claimed") + res, ok := secondOutcome(t, worker) + require.True(t, ok) + assert.Equal(t, FinalizeSaveInProgress, res.Outcome) + assert.Len(t, loop.pendingCRs, 1, "the refused save holds nothing") + }) + + t.Run("while the first waits for a window", func(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + serviceAttach(loop, attachReq("alice", 60*time.Second)) - second := attachReq("alice", 60*time.Second) - second.Name = "save-second" - second.UID = "uid-save-second" - serviceAttach(loop, second) + serviceAttach(loop, second()) + + res, ok := secondOutcome(t, worker) + require.True(t, ok) + assert.Equal(t, FinalizeSaveInProgress, res.Outcome) + }) - pending := loop.pendingCRs[commitRequestID{Namespace: second.Namespace, Name: second.Name, UID: second.UID}] - require.NotNil(t, pending) - assert.False(t, pending.sawForeignWindow, - "a window this author could have claimed is contention, never a mismatch") + t.Run("another author's save is not refused", func(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + serviceAttach(loop, attachReq("alice", 60*time.Second)) + + bob := second() + bob.Author = "bob" + serviceAttach(loop, bob) + + _, resolved := secondOutcome(t, worker) + assert.False(t, resolved, "bob waits for his own window") + assert.Len(t, loop.pendingCRs, 2) + }) + + t.Run("a save for the Next window closes the collecting one instead", func(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + loop.lastPushAt = time.Now() + defer loop.stopTimers() + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, + }}) + serviceAttach(loop, attachReq("alice", 60*time.Second)) + require.NotNil(t, loop.openWindow.pendingCR, "precondition: the first save collects the window") + + next := second() + next.Attach = configv1alpha3.AttachNext + serviceAttach(loop, next) + + _, resolved := secondOutcome(t, worker) + assert.False(t, resolved, "it waits for the next window") + assert.Nil(t, loop.openWindow, "the first save's window was closed under its own message") + }) } // A window that opens AFTER the grace has already elapsed was never a refusal: the request waited diff --git a/internal/git/generator_write.go b/internal/git/generator_write.go index 9d243e906..bfbd3fd0f 100644 --- a/internal/git/generator_write.go +++ b/internal/git/generator_write.go @@ -69,6 +69,7 @@ func (wb *writeBatch) writeGenerated( SourcePath: gen.GeneratedIn, Kind: desired.GetKind(), Name: desired.GetName(), + Namespace: gen.Rendered.GetNamespace(), Desired: desired, Governed: true, }) @@ -139,6 +140,7 @@ func (wb *writeBatch) removeGenerated( SourcePath: gen.GeneratedIn, Kind: obj.GetKind(), Name: obj.GetName(), + Namespace: gen.Rendered.GetNamespace(), Removed: true, }) wb.putToKustomize = true diff --git a/internal/git/generator_write_test.go b/internal/git/generator_write_test.go index efb44648e..68e5a3e01 100644 --- a/internal/git/generator_write_test.go +++ b/internal/git/generator_write_test.go @@ -251,3 +251,38 @@ func TestResync_SweepsAGeneratedConfigMapTheClusterNoLongerHas(t *testing.T) { assertFileBytes(t, envPath, generatorFlagsEnv, "onEvent never sweeps") }) } + +// Two generated ConfigMaps deleted in one batch: the second entry moved up when the first went, and +// its removal still finds it, so both entries and their env files go. +func TestPlanFlush_TwoDeletedGeneratedConfigMapsInOneBatch(t *testing.T) { + worktree := newWorktreeForTest(t) + root := worktree.Filesystem().Root() + seedFile(t, root, "kustomization.yaml", `configMapGenerator: + - name: coffee-flags + envs: + - flags.env + - name: coffee-menu + envs: + - menu.env +generatorOptions: + disableNameSuffixHash: true +`) + seedFile(t, root, "flags.env", "ENABLE_VOUCHERS=true\n") + seedFile(t, root, "menu.env", "ESPRESSO=275\n") + commitCorpusWorktree(t, worktree, "seed two generators") + deleted := func(name string) Event { + return Event{ + Identifier: types.NewResourceIdentifier("", "v1", "configmaps", "coffee", name), + Operation: "DELETE", + } + } + + changed, err := flushEventsForTest(t, newContentWriter(types.SensitiveResourcePolicy{}), worktree, corpusMapper(), + deleted("coffee-flags"), deleted("coffee-menu")) + require.NoError(t, err) + assert.True(t, changed) + assert.NoFileExists(t, filepath.Join(root, "flags.env")) + assert.NoFileExists(t, filepath.Join(root, "menu.env")) + assertFileBytes(t, filepath.Join(root, "kustomization.yaml"), + "generatorOptions:\n disableNameSuffixHash: true\n", "both entries are gone") +} diff --git a/internal/git/manifestedit/kustomization.go b/internal/git/manifestedit/kustomization.go index d4b0b7e67..621efd149 100644 --- a/internal/git/manifestedit/kustomization.go +++ b/internal/git/manifestedit/kustomization.go @@ -482,33 +482,32 @@ type LiteralEdit struct { Remove bool } -// EditConfigMapGeneratorLiterals applies literal edits to configMapGenerator[entryIndex], which -// must be named entryName, preserving comments and key order. An item is edited in place, so its -// position in the list and every other item keep their bytes. All-or-nothing like its siblings: -// a drifted entry skips the whole call with a diagnostic. +// GeneratorEntryID names a configMapGenerator entry by its own name and namespace fields, as +// kustomize names what it generates: two entries may share a name in different namespaces. +type GeneratorEntryID struct { + Name, Namespace string +} + +// EditConfigMapGeneratorLiterals applies literal edits to the configMapGenerator entry id names, +// preserving comments and key order. An item is edited in place, so its position in the list and +// every other item keep their bytes. All-or-nothing like its siblings: an entry the file does not +// hold exactly once skips the whole call with a diagnostic. func EditConfigMapGeneratorLiterals( path string, content []byte, - entryIndex int, - entryName string, + id GeneratorEntryID, edits []LiteralEdit, ) (EditResult, []Diagnostic) { - return editGeneratorEntry(path, content, entryIndex, entryName, func(_, entry *yaml.Node) error { - return applyLiteralEdits(entry, edits) + return editGeneratorEntry(path, content, id, func(_, seq *yaml.Node, index int) error { + return applyLiteralEdits(seq.Content[index], edits) }) } -// RemoveConfigMapGeneratorEntry drops configMapGenerator[entryIndex], which must be named -// entryName. An emptied configMapGenerator: drops the key, as an emptied patches: does. -func RemoveConfigMapGeneratorEntry( - path string, - content []byte, - entryIndex int, - entryName string, -) (EditResult, []Diagnostic) { - return editGeneratorEntry(path, content, entryIndex, entryName, func(root, _ *yaml.Node) error { - seq := nodeMapGet(root, configMapGeneratorKey) - seq.Content = append(seq.Content[:entryIndex], seq.Content[entryIndex+1:]...) +// RemoveConfigMapGeneratorEntry drops the configMapGenerator entry id names. An emptied +// configMapGenerator: drops the key, as an emptied patches: does. +func RemoveConfigMapGeneratorEntry(path string, content []byte, id GeneratorEntryID) (EditResult, []Diagnostic) { + return editGeneratorEntry(path, content, id, func(root, seq *yaml.Node, index int) error { + seq.Content = append(seq.Content[:index], seq.Content[index+1:]...) if len(seq.Content) == 0 { nodeMapDelete(root, configMapGeneratorKey) } @@ -518,14 +517,14 @@ func RemoveConfigMapGeneratorEntry( const configMapGeneratorKey = "configMapGenerator" -// editGeneratorEntry locates configMapGenerator[entryIndex], verifies its name, runs edit on it, -// and re-encodes the kustomization. +// editGeneratorEntry finds the one configMapGenerator entry id names in content as it stands, runs +// edit on it, and re-encodes the kustomization. The entry is found by identity, not by position, so +// an earlier edit of the same file, such as a removed entry before it, cannot point it elsewhere. func editGeneratorEntry( path string, content []byte, - entryIndex int, - entryName string, - edit func(root, entry *yaml.Node) error, + id GeneratorEntryID, + edit func(root, seq *yaml.Node, index int) error, ) (EditResult, []Diagnostic) { skip := func(format string, args ...interface{}) (EditResult, []Diagnostic) { return EditResult{Content: content, Mode: EditSkipped}, @@ -537,17 +536,24 @@ func editGeneratorEntry( } target := docs[idx].body seq := nodeMapGet(root, configMapGeneratorKey) - if seq == nil || seq.Kind != yaml.SequenceNode || entryIndex < 0 || entryIndex >= len(seq.Content) { - return skip("kustomization %s has no configMapGenerator entry %d", path, entryIndex) + if seq == nil || seq.Kind != yaml.SequenceNode { + return skip("kustomization %s has no configMapGenerator", path) } - entry := seq.Content[entryIndex] - if entry.Kind != yaml.MappingNode { - return skip("kustomization %s: configMapGenerator entry %d is not a mapping", path, entryIndex) + index := -1 + for i, entry := range seq.Content { + if entry.Kind != yaml.MappingNode || scalarField(entry, "name") != id.Name || + scalarField(entry, "namespace") != id.Namespace { + continue + } + if index >= 0 { + return skip("kustomization %s holds more than one configMapGenerator entry %s", path, id) + } + index = i } - if name := nodeMapGet(entry, "name"); name == nil || name.Value != entryName { - return skip("kustomization %s: configMapGenerator entry %d is not named %q", path, entryIndex, entryName) + if index < 0 { + return skip("kustomization %s holds no configMapGenerator entry %s", path, id) } - if err := edit(root, entry); err != nil { + if err := edit(root, seq, index); err != nil { return skip("kustomization %s: %v", path, err) } encoded, err := encodeNode(root) @@ -562,6 +568,22 @@ func editGeneratorEntry( return EditResult{Content: []byte(joinDocuments(docs)), Mode: EditPatched}, nil } +// String names the entry in a diagnostic, with its namespace when it has one. +func (id GeneratorEntryID) String() string { + if id.Namespace == "" { + return fmt.Sprintf("named %q", id.Name) + } + return fmt.Sprintf("named %q in namespace %q", id.Name, id.Namespace) +} + +// scalarField is the value of a mapping's scalar field, "" when it has none. +func scalarField(mapping *yaml.Node, key string) string { + if v := nodeMapGet(mapping, key); v != nil && v.Kind == yaml.ScalarNode { + return v.Value + } + return "" +} + // applyLiteralEdits edits an entry's literals: items, creating the list for a first literal. func applyLiteralEdits(entry *yaml.Node, edits []LiteralEdit) error { seq := nodeMapGet(entry, "literals") diff --git a/internal/git/manifestedit/kustomization_test.go b/internal/git/manifestedit/kustomization_test.go index 1fa7ebae7..24633e6d4 100644 --- a/internal/git/manifestedit/kustomization_test.go +++ b/internal/git/manifestedit/kustomization_test.go @@ -365,24 +365,54 @@ func TestAppendKustomizationResource_RefusalsLeaveContentUntouched(t *testing.T) } } -// A configMapGenerator edit pins the entry by index AND name, so a drifted file is skipped rather -// than mis-edited, exactly as an images: edit is. -func TestConfigMapGeneratorEdits_SkipADriftedEntry(t *testing.T) { +// A configMapGenerator edit finds its entry by name and namespace in the bytes as they stand, so an +// entry the file does not hold, or holds twice, is skipped rather than mis-edited. +func TestConfigMapGeneratorEdits_SkipAnEntryNotHeldOnce(t *testing.T) { content := []byte("configMapGenerator:\n - name: menu\n literals:\n - A=1\n") - res, _ := EditConfigMapGeneratorLiterals("kustomization.yaml", content, 0, "flags", + res, _ := EditConfigMapGeneratorLiterals("kustomization.yaml", content, GeneratorEntryID{Name: "flags"}, []LiteralEdit{{Key: "A", Value: "2"}}) require.Equal(t, EditSkipped, res.Mode) - res, _ = RemoveConfigMapGeneratorEntry("kustomization.yaml", content, 1, "menu") + res, _ = RemoveConfigMapGeneratorEntry("kustomization.yaml", content, + GeneratorEntryID{Name: "menu", Namespace: "eu"}) + require.Equal(t, EditSkipped, res.Mode, "the entry has no namespace") + twice := []byte("configMapGenerator:\n - name: menu\n - name: menu\n") + res, _ = RemoveConfigMapGeneratorEntry("kustomization.yaml", twice, GeneratorEntryID{Name: "menu"}) require.Equal(t, EditSkipped, res.Mode) - res, _ = EditConfigMapGeneratorLiterals("kustomization.yaml", content, 0, "menu", + res, _ = EditConfigMapGeneratorLiterals("kustomization.yaml", content, GeneratorEntryID{Name: "menu"}, []LiteralEdit{{Key: "A", Remove: true}, {Key: "Z", Remove: true}}) require.Equal(t, EditPatched, res.Mode) require.Equal(t, "configMapGenerator:\n - name: menu\n", string(res.Content), "an emptied literals: goes, and removing an absent key is a no-op") } +// TestConfigMapGeneratorEdits_FollowAnEarlierEditOfTheFile: removing the first of two entries moves +// the second up, and the edits that follow still find it; entries of one name in two namespaces are +// told apart. +func TestConfigMapGeneratorEdits_FollowAnEarlierEditOfTheFile(t *testing.T) { + content := []byte(`configMapGenerator: + - name: menu + namespace: eu + literals: [A=1] + - name: menu + namespace: us + literals: [A=1] + - name: flags + literals: [B=1] +`) + res, diags := RemoveConfigMapGeneratorEntry("kustomization.yaml", content, + GeneratorEntryID{Name: "menu", Namespace: "eu"}) + require.Equal(t, EditPatched, res.Mode, "%v", diags) + res, diags = EditConfigMapGeneratorLiterals("kustomization.yaml", res.Content, GeneratorEntryID{Name: "flags"}, + []LiteralEdit{{Key: "B", Value: "2"}}) + require.Equal(t, EditPatched, res.Mode, "%v", diags) + res, diags = RemoveConfigMapGeneratorEntry("kustomization.yaml", res.Content, + GeneratorEntryID{Name: "menu", Namespace: "us"}) + require.Equal(t, EditPatched, res.Mode, "%v", diags) + require.Equal(t, "configMapGenerator:\n - name: flags\n literals: [B=2]\n", string(res.Content)) +} + // Comments in a literals: list are the author's, and an edit keeps them: an edited item keeps its // head and line comments, a removed item takes only its own, and a comment below the list stays // below it whichever item ends up last. @@ -442,8 +472,12 @@ func TestEditConfigMapGeneratorLiterals_KeepsComments(t *testing.T) { } for name, tc := range cases { t.Run(name, func(t *testing.T) { - res, diags := EditConfigMapGeneratorLiterals("kustomization.yaml", []byte(in), 0, "branding", - []LiteralEdit{tc.edit}) + res, diags := EditConfigMapGeneratorLiterals( + "kustomization.yaml", + []byte(in), + GeneratorEntryID{Name: "branding"}, + []LiteralEdit{tc.edit}, + ) require.Equal(t, EditPatched, res.Mode, "%v", diags) require.Equal(t, tc.want, string(res.Content)) }) diff --git a/internal/git/save_apply.go b/internal/git/save_apply.go new file mode 100644 index 000000000..e9149957e --- /dev/null +++ b/internal/git/save_apply.go @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "maps" + "strings" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// A CommitRequest with catchUp: AfterPush asks its GitTarget's applier to apply the tip as soon as its +// commit is pushed, and the controller reports how that went in its Applied condition. The save's own +// outcome is decided by the push alone; this is reported beside it and never changes it. +// +// The worker keeps, per applier, the saves waiting for an apply, each with the commit it pushed. The +// attempt that goes out carries them, and the one that succeeds settles each save whose commit the +// applied revision holds (settleSaves). An attempt that does not succeed gives them back, and the next +// carries them again. A save that pushed no commit, whose GitTarget names no applier, or whose applier +// stops being named is settled as applied: no apply is needed, and the cluster, where the saved objects +// came from, already holds them. Only a save whose commit the branch lost is settled as not applied. + +// Why a save's apply was settled, or why it still waits: the Applied condition's reasons. +const ( + SaveApplyApplying = "Applying" + SaveApplyRetrying = "Retrying" + SaveApplyApplied = "Applied" + SaveApplyNothingToApply = "NothingToApply" + SaveApplyNoApplier = "NoApplierClaim" + SaveApplyApplierGone = "ApplierGone" + SaveApplyCommitDropped = "CommitDropped" +) + +// SaveApply is how the apply a CommitRequest asked for stands. +type SaveApply struct { + // Done is whether it is settled: it changes no more. + Done bool + // Applied is whether the applier applied a commit that holds the save's, or no applier applies + // the branch. + Applied bool + // Reason is one of the SaveApply* reasons, and Message says more. + Reason string + Message string + // AppliedRevision is the revision the applier reported applying, once Applied. + AppliedRevision string +} + +type saveApplyEntry struct { + apply SaveApply + updatedAt time.Time +} + +// recordSaveApply stores where a save's apply stands, for the controller to read. The event loop is +// the only writer; the entries are collected with the CommitRequest outcomes. +func (w *BranchWorker) recordSaveApply(id commitRequestID, apply SaveApply) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + if w.crApplies == nil { + w.crApplies = map[commitRequestID]saveApplyEntry{} + } + w.crApplies[id] = saveApplyEntry{apply: apply, updatedAt: time.Now()} +} + +// LookupSaveApply reports where the apply a CommitRequest asked for stands, and false while the +// worker knows nothing of it. +func (w *BranchWorker) LookupSaveApply(namespace, name, uid string) (SaveApply, bool) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + entry, ok := w.crApplies[commitRequestID{Namespace: namespace, Name: name, UID: uid}] + return entry.apply, ok +} + +// collectSaveApplies drops the entries older than the outcome TTL. The caller holds crOutcomesMu. +func (w *BranchWorker) collectSaveApplies(now time.Time) { + for id, entry := range w.crApplies { + if now.Sub(entry.updatedAt) > commitRequestOutcomeTTL { + delete(w.crApplies, id) + } + } +} + +// catchUpAfterSave is called as a CommitRequest that asked to catch up resolves. With a pushed +// commit, its GitTarget's applier gets a converge when the appliers are next listed, at the end of +// this wake (takeSavedCatchUps); with none, there is nothing to apply. +func (l *branchWorkerEventLoop) catchUpAfterSave(pcr *pendingCommitRequest, commit string) { + if commit == "" { + l.w.recordSaveApply(pcr.id, SaveApply{Done: true, Applied: true, Reason: SaveApplyNothingToApply, + Message: "the request pushed no commit: what it saved was in Git already, and the cluster holds it"}) + return + } + if l.savedCatchUps == nil { + l.savedCatchUps = map[itypes.ResourceReference]map[commitRequestID]string{} + } + ref := itypes.NewResourceReference(pcr.gitTargetName, pcr.gitTargetNamespace) + if l.savedCatchUps[ref] == nil { + l.savedCatchUps[ref] = map[commitRequestID]string{} + } + l.savedCatchUps[ref][pcr.id] = commit + l.w.recordSaveApply(pcr.id, SaveApply{Reason: SaveApplyApplying, + Message: "commit " + commit + " is pushed; its applier is asked to apply it"}) +} + +// takeSavedCatchUps leaves a converge pending for every applier a pushed CommitRequest asked to catch +// up, and records the request as waiting on it. It was asked for, so it is no lag converge: it does +// not yield to an edit (convergeHolds). A request whose GitTarget names no applier is settled. +func (l *branchWorkerEventLoop) takeSavedCatchUps() { + for ref, saves := range l.savedCatchUps { + state := l.applierNaming(ref) + if state == nil { + for id := range saves { + l.w.recordSaveApply(id, SaveApply{Done: true, Applied: true, Reason: SaveApplyNoApplier, + Message: "GitTarget " + ref.String() + " names no applier claim, so nothing applies the " + + "branch; the cluster already holds the saved objects"}) + } + continue + } + state.converge, state.lag = true, false + for id, commit := range saves { + state.waitSave(id, commit) + } + l.w.Log.Info("A CommitRequest asked its applier to catch up with the pushed commit", + "branch", l.w.Branch, "applier", state.ref.String(), "gitTarget", ref.String(), "requests", len(saves)) + } + l.savedCatchUps = nil +} + +// applierNaming is the applier a GitTarget names, as last listed, or nil. +func (l *branchWorkerEventLoop) applierNaming(ref itypes.ResourceReference) *applierState { + for _, state := range l.appliers { + for _, target := range state.targets { + if target.ref == ref { + return state + } + } + } + return nil +} + +// waitSave records a CommitRequest, which pushed commit, as waiting for the applier's next apply that +// succeeds. +func (s *applierState) waitSave(id commitRequestID, commit string) { + if s.waitingSaves == nil { + s.waitingSaves = map[commitRequestID]string{} + } + s.waitingSaves[id] = commit +} + +// settleSaves settles the saves an apply of revision answered, by what revision holds: +// +// - a save whose commit revision names or descends from is applied; +// - one whose commit the branch no longer holds either is not, and never will be: the branch's +// history was rewritten after the push; +// - one whose commit the branch holds and revision does not waits for the converge left pending; +// - one the checkout cannot tell about, because it lacks a commit, waits as well, and the remote is +// looked at: revision is a commit pushed elsewhere, and its fetch brings the apply that settles it. +func (l *branchWorkerEventLoop) settleSaves(state *applierState, saves map[commitRequestID]string, revision string) { + if len(saves) == 0 { + return + } + repo, err := gogit.PlainOpen(l.w.repoPath()) + if err != nil { + repo = nil + } + applied, tip := revisionCommit(revision), l.w.inbox.lastTip() + look := false + for id, commit := range saves { + held, known := commitHolds(repo, applied, commit) + if held { + l.w.recordSaveApply(id, SaveApply{Done: true, Applied: true, Reason: SaveApplyApplied, + AppliedRevision: revision, Message: "the applier applied " + revision}) + continue + } + onBranch, branchKnown := commitHolds(repo, tip, commit) + switch { + case known && branchKnown && !onBranch: + l.w.recordSaveApply(id, SaveApply{Done: true, Reason: SaveApplyCommitDropped, Message: "the applier " + + "applied " + revision + ", and the branch no longer holds commit " + commit + ": its history was " + + "rewritten after the push"}) + continue + case known: + state.converge, state.lag = true, false + l.w.recordSaveApply(id, SaveApply{ + Reason: SaveApplyRetrying, + Message: "the applier applied " + revision + ", which does not hold commit " + commit + "; asking again", + }) + default: + look = look || applied != "" + l.w.recordSaveApply(id, SaveApply{Reason: SaveApplyRetrying, Message: "the applier applied " + revision + + ", and whether it holds commit " + commit + " is not known here yet; looking at the remote"}) + } + state.waitSave(id, commit) + } + if look { + l.w.EnqueuePushNotification(applied) + } +} + +// commitHolds reports whether descendant is commit or descends from it, and whether the checkout could +// tell: it holds both commits. +func commitHolds(repo *gogit.Repository, descendant, commit string) (bool, bool) { + if descendant == "" || commit == "" { + return false, false + } + if strings.EqualFold(descendant, commit) { + return true, true + } + if repo == nil { + return false, false + } + descendantID, ok := plumbing.FromHex(descendant) + if !ok { + return false, false + } + commitID, ok := plumbing.FromHex(commit) + if !ok { + return false, false + } + later, err := repo.CommitObject(descendantID) + if err != nil { + return false, false + } + earlier, err := repo.CommitObject(commitID) + if err != nil { + return false, false + } + held, err := earlier.IsAncestor(later) + if err != nil { + return false, false + } + return held, true +} + +// retrySaves records the saves an attempt that did not succeed gave back as waiting again, with why. +func (l *branchWorkerEventLoop) retrySaves(saves map[commitRequestID]string, why string) { + for id := range saves { + l.w.recordSaveApply(id, SaveApply{Reason: SaveApplyRetrying, + Message: "the apply did not complete, and is asked again: " + why}) + } +} + +// dropSaves settles the saves of an applier no GitTarget names any more as applied: nothing applies the +// branch for them now, and the cluster already holds the saved objects. +func (l *branchWorkerEventLoop) dropSaves(state *applierState) { + saves := maps.Clone(state.waitingSaves) + if state.inflight != nil { + if saves == nil { + saves = map[commitRequestID]string{} + } + maps.Copy(saves, state.inflight.saves) + } + for id := range saves { + l.w.recordSaveApply(id, SaveApply{Done: true, Applied: true, Reason: SaveApplyApplierGone, + Message: "applier " + state.ref.String() + " is no longer named on this branch, so nothing applies " + + "it; the cluster already holds the saved objects"}) + } +} diff --git a/internal/git/types.go b/internal/git/types.go index 47059fa83..392fc6ad9 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -458,6 +458,9 @@ type WorkItem struct { // Withdraw cancels a CommitRequest the worker has not acted on yet. It rides the same FIFO as // the attach, so it is always handled after every attach sent before it. Withdraw *AttachCommitRequest + // ApplyBarrier is one answer to the barrier an apply window took (apply_barrier.go). A stream's + // answer rides the FIFO behind every event that stream routed before it. + ApplyBarrier *ApplyBarrier // charge is what a write or an attach was charged against the intake budget at enqueue, and is // released once the loop has handled it. A resync's is on its marker, because coalescing can diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index 848ef12b2..16d49f00e 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -185,6 +185,9 @@ type WorkerManager struct { // tests. initialSnapshots func(target types.ResourceReference) bool resyncRecheck func(target types.ResourceReference) + // applyBarrier lets every worker's apply windows wait for the watch (apply_barrier.go). Set once + // at startup (SetApplyBarrier); nil in the CLI and in tests. + applyBarrier ApplyBarrierFunc // applyRequester delivers every worker's apply requests. Set once at startup // (SetApplyRequester) before any worker is created; nil in the CLI and in tests, which switches @@ -279,6 +282,15 @@ func (m *WorkerManager) SetSnapshotHooks( m.initialSnapshots, m.resyncRecheck = settled, recheck } +// SetApplyBarrier injects the watch side's barrier, which lets an apply window wait until the watch +// delivered every write made before its apply completed (apply_barrier.go). Like SetMapper, it is +// called once at startup before any worker is created. +func (m *WorkerManager) SetApplyBarrier(barrier ApplyBarrierFunc) { + m.mu.Lock() + defer m.mu.Unlock() + m.applyBarrier = barrier +} + // SetSSHHostKeyConfig injects the SSH host-key resolution config used by every worker's credential // reads. Like SetMapper, it is called once at startup before any worker is created. func (m *WorkerManager) SetSSHHostKeyConfig(cfg SSHHostKeyConfig) { @@ -522,6 +534,7 @@ func (m *WorkerManager) startWorkerLocked(key BranchKey, req WorkerRequest, gitB worker.claimStatus = m.claimStatus worker.initialSnapshots = m.initialSnapshots worker.resyncRecheck = m.resyncRecheck + worker.applyBarrier = m.applyBarrier worker.replacementPending = func() bool { return m.ReplacementPending(key) } worker.applierAdmitted = m.applierAdmitted worker.renderFidelityGate = m.renderFidelityGate diff --git a/internal/manifestanalyzer/generator_write.go b/internal/manifestanalyzer/generator_write.go index 0d51e1a99..0e58e4887 100644 --- a/internal/manifestanalyzer/generator_write.go +++ b/internal/manifestanalyzer/generator_write.go @@ -53,8 +53,13 @@ type GeneratedObject struct { // Roots are the render roots that render this entry, in any namespace, sorted. Roots []string // SharedInputs are the entry's input files something else also reads: another generator - // entry, a resources: entry or a patch. A delete of the object keeps them. + // entry, a resources: entry, a patch or a release's values files. A delete of the object + // keeps them. SharedInputs map[string]struct{} + // ValuesInputs are the entry's input files a release names as a values file: an Argo CD + // Application's helm.valueFiles or a Flux HelmRelease's valuesFiles. A write to one is + // refused, since the render shows nothing of what it does to the release. + ValuesInputs map[string]struct{} // Resource is the API-side identity of the rendered object, as a document's ResourceIdentity // is: set only when the registry resolves its GVK to a followable resource. The resync sweep // drops only a generated object it can name this way. @@ -109,8 +114,13 @@ type generatedRender struct { } // generatedObjects groups the generated renders of a scan into one GeneratedObject per entry and -// namespace, each naming every root that renders the entry. -func generatedObjects(renders []generatedRender, kusts map[string]*kustomizationDoc) []*GeneratedObject { +// namespace, each naming every root that renders the entry. valuesFiles are the files a release +// names as a values file (helmValueFileRefs). +func generatedObjects( + renders []generatedRender, + kusts map[string]*kustomizationDoc, + valuesFiles map[string]struct{}, +) []*GeneratedObject { type entryKey struct{ in, kind, name string } type objectKey struct { entry entryKey @@ -118,7 +128,7 @@ func generatedObjects(renders []generatedRender, kusts map[string]*kustomization } roots := map[entryKey]map[string]struct{}{} byObject := map[objectKey]*GeneratedObject{} - inputUses := inputUseCounts(kusts) + inputUses := inputUseCounts(kusts, valuesFiles) var order []objectKey for _, r := range renders { ek := entryKey{in: r.in, kind: r.object.GetKind(), name: r.object.GetName()} @@ -136,6 +146,7 @@ func generatedObjects(renders []generatedRender, kusts map[string]*kustomization Entry: entry, Rendered: r.object, SharedInputs: sharedInputsOf(entry, inputUses), + ValuesInputs: valuesInputsOf(entry, valuesFiles), } order = append(order, ok) } @@ -149,8 +160,8 @@ func generatedObjects(renders []generatedRender, kusts map[string]*kustomization } // inputUseCounts counts, per file, how many generator entries, resources: entries and patches of -// the scan read it. -func inputUseCounts(kusts map[string]*kustomizationDoc) map[string]int { +// the scan read it, and one more for a file a release names as a values file. +func inputUseCounts(kusts map[string]*kustomizationDoc, valuesFiles map[string]struct{}) map[string]int { uses := map[string]int{} for _, doc := range kusts { for _, entry := range doc.configMapGenerators { @@ -169,6 +180,9 @@ func inputUseCounts(kusts map[string]*kustomizationDoc) map[string]int { for p := range reachedResourceFiles(kusts) { uses[p]++ } + for p := range valuesFiles { + uses[p]++ + } return uses } @@ -186,6 +200,20 @@ func sharedInputsOf(entry *ConfigMapGeneratorEntry, uses map[string]int) map[str return out } +// valuesInputsOf is the entry's inputs a release names as a values file. +func valuesInputsOf(entry *ConfigMapGeneratorEntry, valuesFiles map[string]struct{}) map[string]struct{} { + if entry == nil { + return nil + } + out := map[string]struct{}{} + for _, p := range entry.inputPaths() { + if _, ok := valuesFiles[p]; ok { + out[p] = struct{}{} + } + } + return out +} + // generatorEntryFor finds the entry that rendered obj: the one entry of its name, or, when two // entries share a name, the one whose own namespace is the object's. func generatorEntryFor(doc *kustomizationDoc, obj *unstructured.Unstructured) *ConfigMapGeneratorEntry { @@ -250,12 +278,19 @@ var errGeneratedWrite = errors.New("no destination for the change") // the inputs of the generator that renders gen. writeScope is the write jail (empty: everything // is writable), and read returns an input file's current bytes. An error is a refusal and names // the reason; nothing has been written. +// +// A desired object the generator already renders plans nothing, and so needs no input it may +// write: a generator in a read-only base, or one two render roots share, is matched as it stands. +// The oracle still proves the render equals the live object. func PlanGeneratedWrite( gen *GeneratedObject, desired *unstructured.Unstructured, writeScope string, read func(path string) ([]byte, bool), ) ([]GeneratedWrite, error) { + if rendersUnchanged(gen.Rendered, desired) { + return nil, nil + } entry, err := writableEntry(gen, writeScope) if err != nil { return nil, err @@ -275,7 +310,9 @@ func PlanGeneratedWrite( errGeneratedWrite) } - plan := generatorPlan{entry: entry, writeScope: writeScope, read: read, files: map[string][]byte{}} + plan := generatorPlan{ + entry: entry, writeScope: writeScope, valuesInputs: gen.ValuesInputs, read: read, files: map[string][]byte{}, + } liveData, _, _ := unstructured.NestedStringMap(desired.Object, "data") for _, key := range sortedKeysOfSet(unionStringKeys(liveData, current.data)) { if err := plan.key(key, liveData, current); err != nil { @@ -288,6 +325,22 @@ func PlanGeneratedWrite( return plan.writes(read), nil } +// rendersUnchanged reports whether rendered, the generator's render as Git stands, already is +// desired, setting the namespace aside as changedOutsideData does: the oracle compares it. +func rendersUnchanged(rendered, desired *unstructured.Unstructured) bool { + if len(changedOutsideData(desired, rendered)) > 0 { + return false + } + for _, field := range []string{"data", "binaryData"} { + want, _, _ := unstructured.NestedStringMap(desired.Object, field) + have, _, _ := unstructured.NestedStringMap(rendered.Object, field) + if !equalStringMaps(want, have) { + return false + } + } + return true +} + // writableEntry is the entry behind gen, or the refusal that says why its inputs are not this // target's to write: no single entry, an entry two render roots share, or one in a read-only base. func writableEntry(gen *GeneratedObject, writeScope string) (*ConfigMapGeneratorEntry, error) { @@ -324,8 +377,7 @@ func PlanGeneratedDelete( if !ok { return nil, fmt.Errorf("%w: cannot read %s", errGeneratedWrite, entry.KustomizationPath) } - res, diags := manifestedit.RemoveConfigMapGeneratorEntry( - entry.KustomizationPath, kustomization, entry.Index, entry.Name) + res, diags := manifestedit.RemoveConfigMapGeneratorEntry(entry.KustomizationPath, kustomization, entry.ID()) if res.Mode != manifestedit.EditPatched { return nil, fmt.Errorf("%w: the entry cannot be removed from %s%s", errGeneratedWrite, entry.KustomizationPath, diagnosticSuffix(diags)) @@ -366,8 +418,10 @@ func diagnosticSuffix(diags []manifestedit.Diagnostic) string { type generatorPlan struct { entry *ConfigMapGeneratorEntry writeScope string - read func(string) ([]byte, bool) - files map[string][]byte + // valuesInputs are the inputs a release names as a values file, which no write may touch. + valuesInputs map[string]struct{} + read func(string) ([]byte, bool) + files map[string][]byte // literals are the literals: edits, applied to the kustomization in one call at the end. literals []manifestedit.LiteralEdit } @@ -429,8 +483,7 @@ func (p *generatorPlan) applyLiterals() error { return nil } path := p.entry.KustomizationPath - res, diags := manifestedit.EditConfigMapGeneratorLiterals( - path, p.content(path), p.entry.Index, p.entry.Name, p.literals) + res, diags := manifestedit.EditConfigMapGeneratorLiterals(path, p.content(path), p.entry.ID(), p.literals) switch res.Mode { case manifestedit.EditPatched: p.files[path] = res.Content @@ -469,11 +522,16 @@ func (p *generatorPlan) setEnv(path, key, value string) error { return nil } -// writable refuses an input file outside the write jail. +// writable refuses an input file outside the write jail, and one a release reads as a values file: +// the oracle renders the kustomization, and cannot see what the edit does to the release. func (p *generatorPlan) writable(path string) error { if p.writeScope != "" && !pathWithin(path, p.writeScope) { return fmt.Errorf("%w: %s is outside spec.path", errGeneratedWrite, path) } + if _, values := p.valuesInputs[path]; values { + return fmt.Errorf("%w: %s is also a release's values file, which the render cannot check", + errGeneratedWrite, path) + } return nil } diff --git a/internal/manifestanalyzer/generators.go b/internal/manifestanalyzer/generators.go index 625d397be..f338a1377 100644 --- a/internal/manifestanalyzer/generators.go +++ b/internal/manifestanalyzer/generators.go @@ -35,8 +35,6 @@ const featureGeneratorBehavior = "configMapGenerator-behavior" type ConfigMapGeneratorEntry struct { // KustomizationPath is the kustomization file declaring the entry (slash). KustomizationPath string - // Index is the entry's position in configMapGenerator:. - Index int // Name and Namespace are the entry's own fields. Namespace is usually empty: a namespace: // transformer supplies it. Name, Namespace string @@ -55,6 +53,12 @@ type GeneratorFileSource struct { Path string } +// ID names the entry in its kustomization: by its name and namespace, which an edit of the file +// leaves as they are, where its position may move. +func (e ConfigMapGeneratorEntry) ID() manifestedit.GeneratorEntryID { + return manifestedit.GeneratorEntryID{Name: e.Name, Namespace: e.Namespace} +} + // inputPaths lists every file the entry reads. func (e ConfigMapGeneratorEntry) inputPaths() []string { out := make([]string, 0, len(e.Files)+len(e.Envs)) @@ -75,7 +79,7 @@ func configMapGenerators(k kustypes.Kustomization, kustomizationPath string) ([] globalHashOff := k.GeneratorOptions != nil && k.GeneratorOptions.DisableNameSuffixHash features := map[string]struct{}{} var out []ConfigMapGeneratorEntry - for i, g := range k.ConfigMapGenerator { + for _, g := range k.ConfigMapGenerator { switch g.Behavior { case "", "create": // kustomize ORs the two (types.MergeGlobalOptionsIntoLocal): either one disables it. @@ -87,7 +91,7 @@ func configMapGenerators(k kustypes.Kustomization, kustomizationPath string) ([] // it would not make the entry adoptable. features[featureGeneratorBehavior] = struct{}{} } - entry, ok := generatorEntry(g, kustomizationPath, dir, i) + entry, ok := generatorEntry(g, kustomizationPath, dir) if ok { out = append(out, entry) } @@ -97,14 +101,9 @@ func configMapGenerators(k kustypes.Kustomization, kustomizationPath string) ([] // generatorEntry resolves one entry's sources. ok is false when a files: source does not parse // the way kustomize parses it. -func generatorEntry( - g kustypes.ConfigMapArgs, - kustomizationPath, dir string, - index int, -) (ConfigMapGeneratorEntry, bool) { +func generatorEntry(g kustypes.ConfigMapArgs, kustomizationPath, dir string) (ConfigMapGeneratorEntry, bool) { entry := ConfigMapGeneratorEntry{ KustomizationPath: kustomizationPath, - Index: index, Name: g.Name, Namespace: g.Namespace, Literals: append([]string(nil), g.LiteralSources...), diff --git a/internal/manifestanalyzer/generators_test.go b/internal/manifestanalyzer/generators_test.go index 0aca5ef88..20153574b 100644 --- a/internal/manifestanalyzer/generators_test.go +++ b/internal/manifestanalyzer/generators_test.go @@ -406,7 +406,8 @@ func TestLiteralValue_AgreesWithKustomize(t *testing.T) { } kustomization := []byte("configMapGenerator:\n - name: probe\n literals: [SEED=x]\n" + "generatorOptions:\n disableNameSuffixHash: true\n") - res, diags := manifestedit.EditConfigMapGeneratorLiterals("kustomization.yaml", kustomization, 0, "probe", edits) + res, diags := manifestedit.EditConfigMapGeneratorLiterals("kustomization.yaml", kustomization, + manifestedit.GeneratorEntryID{Name: "probe"}, edits) require.Equal(t, manifestedit.EditPatched, res.Mode, "%v", diags) rendered, err := renderRoot([]manifestedit.FileContent{{Path: "kustomization.yaml", Content: res.Content}}, ".") @@ -444,3 +445,133 @@ func TestBuildPlan_SweepsAGeneratedObject(t *testing.T) { structureOnly := buildStoreFS(context.Background(), fsys, nil, WriterAllowlist()) require.Empty(t, drops(structureOnly, allInScope), "a scan with no registry names nothing to drop") } + +// A live object the generator already renders plans no write, so it needs no input it may write: +// a generator two render roots share, or one in a read-only base, is matched as it stands. +func TestPlanGeneratedWrite_AnUnchangedObjectNeedsNoWritableEntry(t *testing.T) { + sharedBase := fstest.MapFS{ + "base/kustomization.yaml": {Data: []byte("configMapGenerator:\n - name: menu\n envs: [menu.env]\n" + + "generatorOptions:\n disableNameSuffixHash: true\n")}, + "base/menu.env": {Data: []byte("A=1\n")}, + "overlays/a/kustomization.yaml": {Data: []byte("resources: [../../base]\n")}, + "overlays/b/kustomization.yaml": {Data: []byte("resources: [../../base]\n")}, + } + cases := map[string]struct { + fsys fstest.MapFS + live *unstructured.Unstructured + writeScope string + }{ + "shared by two render roots": { + fsys: sharedBase, + live: liveConfigMap("", "menu", map[string]interface{}{"A": "1"}), + }, + "in a read-only base": { + fsys: coffeeFS(), + live: liveConfigMap("coffee", "coffee-flags", map[string]interface{}{ + "ENABLE_VOUCHERS": "true", "SHOW_BANNER": "true", + }), + writeScope: "overlays/prod", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + generated := generatedFor(t, tc.fsys, tc.live) + require.NotEmpty(t, generated) + writes, err := PlanGeneratedWrite(generated[0], tc.live, tc.writeScope, func(string) ([]byte, bool) { + t.Fatal("an unchanged object reads no input") + return nil, false + }) + require.NoError(t, err) + require.Empty(t, writes) + }) + } +} + +// An input a HelmRelease also names as a values file is the release's too, and the render cannot +// show what an edit does to the release: a delete keeps it, and a write to it is refused. +func TestGeneratedInputs_AReleasesValuesFileIsKeptAndNeverWritten(t *testing.T) { + fsys := fstest.MapFS{ + "kustomization.yaml": {Data: []byte("resources: [release.yaml]\n" + + "configMapGenerator:\n - {name: chart-values, files: [values.yaml], literals: [MODE=a]}\n" + + "generatorOptions:\n disableNameSuffixHash: true\n")}, + "release.yaml": {Data: []byte("apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\n" + + "metadata: {name: shop}\nspec:\n chart:\n spec:\n chart: shop\n valuesFiles: [values.yaml]\n")}, + "values.yaml": {Data: []byte("replicas: 2\n")}, + } + read := func(p string) ([]byte, bool) { + f, ok := fsys[p] + if !ok { + return nil, false + } + return f.Data, true + } + live := liveConfigMap("", "chart-values", map[string]interface{}{"values.yaml": "replicas: 2\n", "MODE": "a"}) + generated := generatedFor(t, fsys, live) + require.Len(t, generated, 1) + + writes, err := PlanGeneratedDelete(generated[0], "", read) + require.NoError(t, err) + require.Equal(t, []string{"kustomization.yaml"}, pathsOf(toFileContents(writes)), "values.yaml stays") + + _, err = PlanGeneratedWrite( + generated[0], + liveConfigMap( + "", + "chart-values", + map[string]interface{}{"values.yaml": "replicas: 3\n", "MODE": "a"}, + ), + "", + read, + ) + require.ErrorContains(t, err, "values.yaml is also a release's values file") + + writes, err = PlanGeneratedWrite( + generated[0], + liveConfigMap( + "", + "chart-values", + map[string]interface{}{"values.yaml": "replicas: 2\n", "MODE": "b"}, + ), + "", + read, + ) + require.NoError(t, err, "a key of another input is written as before") + require.Equal(t, []string{"kustomization.yaml"}, pathsOf(toFileContents(writes))) +} + +func toFileContents(writes []GeneratedWrite) []manifestedit.FileContent { + out := make([]manifestedit.FileContent, 0, len(writes)) + for _, w := range writes { + out = append(out, manifestedit.FileContent{Path: w.Path, Content: w.Content}) + } + return out +} + +// Two generated objects of one name, from one kustomization, in two namespaces are two objects to +// the oracle: removing one is not refused because the other still renders, and a write to one is +// checked against that one. +func TestVerifyBatchRenders_GeneratedObjectsInTwoNamespaces(t *testing.T) { + kustomization := func(entries string) manifestedit.FileContent { + return manifestedit.FileContent{Path: "kustomization.yaml", Content: []byte("configMapGenerator:\n" + entries + + "generatorOptions:\n disableNameSuffixHash: true\n")} + } + eu := " - {name: menu, namespace: eu, literals: [A=1]}\n" + us := " - {name: menu, namespace: us, literals: [A=1]}\n" + before := []manifestedit.FileContent{kustomization(eu + us)} + + removed := WriteIntent{SourcePath: "kustomization.yaml", Kind: "ConfigMap", Name: "menu", Namespace: "eu", + Removed: true} + require.NoError(t, VerifyBatchRenders(before, []manifestedit.FileContent{kustomization(us)}, + []WriteIntent{removed}, VerifyOptions{})) + + written := liveConfigMap("us", "menu", map[string]interface{}{"A": "2"}) + intent := WriteIntent{SourcePath: "kustomization.yaml", Kind: "ConfigMap", Name: "menu", Namespace: "us", + Desired: written, Governed: true} + after := []manifestedit.FileContent{kustomization(eu + " - {name: menu, namespace: us, literals: [A=2]}\n")} + require.NoError(t, VerifyBatchRenders(before, after, []WriteIntent{intent}, VerifyOptions{})) + + intent.Namespace = "eu" + written.SetNamespace("eu") + require.Error(t, VerifyBatchRenders(before, after, []WriteIntent{intent}, VerifyOptions{}), + "eu did not move, and us moved without an intent") +} diff --git a/internal/manifestanalyzer/override_chain.go b/internal/manifestanalyzer/override_chain.go index bf9ec341d..becafe212 100644 --- a/internal/manifestanalyzer/override_chain.go +++ b/internal/manifestanalyzer/override_chain.go @@ -29,12 +29,18 @@ import ( const reasonRenderFailed manifestedit.DiagReason = "kustomize-build-failed" // chainKey identifies a rendered document: the source file kustomize says produced -// it, plus its kind and name. Namespace is deliberately absent — a namespace -// transformer rewrites it, and the source document may not carry one at all. +// it, plus its kind and name. Namespace is deliberately absent for a document — a +// namespace transformer rewrites it, and the source document may not carry one at all. +// +// A generated object is the exception. Its origin is the kustomization whose generator +// made it, and one kustomization can generate two objects of one name in different +// namespaces (entries with their own namespace:), so namespace, as rendered, tells +// them apart. It is empty for everything else. type chainKey struct { originPath string kind string name string + namespace string } // renderRoots returns the kustomization directories no other kustomization in the diff --git a/internal/manifestanalyzer/render_verify.go b/internal/manifestanalyzer/render_verify.go index 9ed74612f..f3c653584 100644 --- a/internal/manifestanalyzer/render_verify.go +++ b/internal/manifestanalyzer/render_verify.go @@ -49,6 +49,10 @@ type WriteIntent struct { SourcePath string // Kind and Name identify the document within that file. Kind, Name string + // Namespace is set only for a generated object, whose SourcePath is the kustomization that + // generated it: the namespace the render gives it, which tells apart two objects of one name + // that kustomization generates (see chainKey). + Namespace string // Desired is the live object the document must render to. It is the whole point of // the check, and it is nil only when Removed or Unchecked says there is nothing to // check against. @@ -72,7 +76,7 @@ type WriteIntent struct { } func (w WriteIntent) key() chainKey { - return chainKey{originPath: filepathToSlash(w.SourcePath), kind: w.Kind, name: w.Name} + return chainKey{originPath: filepathToSlash(w.SourcePath), kind: w.Kind, name: w.Name, namespace: w.Namespace} } // RenderRefusedError is the oracle's verdict: the bytes the flush was about to commit do @@ -314,11 +318,15 @@ func sameObject(a, b *unstructured.Unstructured) bool { // renderedByKey indexes a render by the document each object came from. The key carries the // ORIGIN FILE, not just kind/name, so two same-named objects rendered from different source // files stay distinct. A generated object is keyed by the kustomization that generated it, which -// is also the SourcePath its write intent names. +// is also the SourcePath its write intent names, and by its namespace. func renderedByKey(objects []renderedObject) map[chainKey]renderedObject { out := make(map[chainKey]renderedObject, len(objects)) for _, o := range objects { - out[chainKey{originPath: o.sourceKey(), kind: o.Object.GetKind(), name: o.Object.GetName()}] = o + key := chainKey{originPath: o.sourceKey(), kind: o.Object.GetKind(), name: o.Object.GetName()} + if o.OriginPath == "" { + key.namespace = o.Object.GetNamespace() + } + out[key] = o } return out } diff --git a/internal/manifestanalyzer/store.go b/internal/manifestanalyzer/store.go index ecd813d44..1a4947b7d 100644 --- a/internal/manifestanalyzer/store.go +++ b/internal/manifestanalyzer/store.go @@ -462,7 +462,9 @@ func buildStore( renderInputs: reachedResourceFiles(kusts), renderedDocuments: renderedDocumentsOf(ovAssignments), GeneratorInputs: pathSet(scan.InputFiles), - generated: generatedObjects(generated, kusts), + // A generator input is out of the YAML files, so the values files it may be are looked + // for among the render's. + generated: generatedObjects(generated, kusts, helmValueFileRefs(scan.RenderFiles())), } store.resolveGeneratedResources(ctx, lookup, settings.declaredNamespace) diff --git a/internal/queue/command_author_store.go b/internal/queue/command_author_store.go deleted file mode 100644 index ad012b485..000000000 --- a/internal/queue/command_author_store.go +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package queue - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/redis/go-redis/v9" - "k8s.io/apimachinery/pkg/types" -) - -// commandAuthorKeySuffix namespaces the keys this store owns. It shares the top-level -// author domain with audit-sourced resource facts, but the command subfamily has a -// different provenance and no grace-window join. -const commandAuthorKeySuffix = ":author:v1:command:" - -// commandAuthorRecordTTL bounds a captured authorship record. It is NOT tuned to cover -// any wait: by the authorship invariant the record is written before the object exists, -// and the controller reads it on the first reconcile after persist — typically -// sub-second. The TTL exists ONLY so an orphan record (a command object deleted before -// its reconcile) self-cleans. It is a fixed internal constant, deliberately not a flag: -// there is nothing to tune. An hour is generous headroom for a slow reconcile backlog or -// a leader failover. -const commandAuthorRecordTTL = time.Hour - -// CommandAuthor is the minimal authorship captured at admission for one command object. -// It carries only what a git commit author needs — no RV, no auditID, no conflict bit: -// this is a 1:1 command capture, not a post-persist join. -type CommandAuthor struct { - Author string `json:"author"` - DisplayName string `json:"displayName,omitempty"` - Email string `json:"email,omitempty"` - RequestedAt string `json:"requestedAt,omitempty"` // RFC3339Nano, for lag metrics/debug -} - -// CommandAuthorStore records and reads command authorship. It shares RedisStore's -// connection but is wired whenever the validate-operator-types webhook is enabled — -// independent of --author-attribution, which only governs mirrored-resource attribution. -type CommandAuthorStore struct { - client *redis.Client - ttl time.Duration - // keyPrefix is the root namespace shared with the RedisStore that built this store. - keyPrefix string -} - -// RecordCommandAuthor is the admission-side write: capture the authenticated submitter -// the instant a command CREATE is admitted, before it persists. Last-write-wins (a -// CREATE fires admission once; a retried admission re-asserts the same user). -func (s *CommandAuthorStore) RecordCommandAuthor( - ctx context.Context, uid types.UID, author CommandAuthor, -) error { - raw, err := json.Marshal(author) - if err != nil { - return fmt.Errorf("marshal command author: %w", err) - } - return s.client.Set(ctx, s.key(uid), raw, s.ttl).Err() -} - -// LookupCommandAuthor is the controller-side read, keyed by the persisted object's UID. -// ok=false means no record was captured — the validate-operator-types webhook is not -// configured (or a best-effort write missed) — and the controller finalizes as the -// committer, immediately. -func (s *CommandAuthorStore) LookupCommandAuthor( - ctx context.Context, uid types.UID, -) (CommandAuthor, bool) { - raw, err := s.client.Get(ctx, s.key(uid)).Bytes() - if err != nil { - return CommandAuthor{}, false - } - var a CommandAuthor - if json.Unmarshal(raw, &a) != nil || a.Author == "" { - return CommandAuthor{}, false - } - return a, true -} - -// key identifies the command by UID alone — globally unique, like the watch cursor key, -// so namespace/name/kind would be redundant (kept out for a tight key). -func (s *CommandAuthorStore) key(uid types.UID) string { - return resolveKeyPrefix(s.keyPrefix) + commandAuthorKeySuffix + escapeKeyField(string(uid)) -} diff --git a/internal/queue/command_author_store_test.go b/internal/queue/command_author_store_test.go deleted file mode 100644 index 289751acd..000000000 --- a/internal/queue/command_author_store_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package queue - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func newTestCommandAuthorStore(t *testing.T) *CommandAuthorStore { - t.Helper() - return newTestRedisStore(t).CommandAuthorStore() -} - -func TestCommandAuthorStore_RecordAndLookup(t *testing.T) { - store := newTestCommandAuthorStore(t) - ctx := context.Background() - - author := CommandAuthor{ - Author: "alice", - DisplayName: "Alice", - Email: "alice@example.com", - RequestedAt: "2026-06-29T00:00:00Z", - } - require.NoError(t, store.RecordCommandAuthor(ctx, "cr-uid", author)) - - got, ok := store.LookupCommandAuthor(ctx, "cr-uid") - require.True(t, ok) - require.Equal(t, author, got) -} - -func TestCommandAuthorStore_LookupMissIsImmediate(t *testing.T) { - store := newTestCommandAuthorStore(t) - - // Present-or-never: an unrecorded uid is an immediate miss, no wait. - _, ok := store.LookupCommandAuthor(context.Background(), "never-recorded") - require.False(t, ok) -} - -func TestCommandAuthorStore_LastWriteWins(t *testing.T) { - store := newTestCommandAuthorStore(t) - ctx := context.Background() - - require.NoError(t, store.RecordCommandAuthor(ctx, "cr-uid", CommandAuthor{Author: "alice"})) - require.NoError(t, store.RecordCommandAuthor(ctx, "cr-uid", CommandAuthor{Author: "bob"})) - - got, ok := store.LookupCommandAuthor(ctx, "cr-uid") - require.True(t, ok) - require.Equal(t, "bob", got.Author) -} - -func TestCommandAuthorStore_EmptyAuthorIsMiss(t *testing.T) { - store := newTestCommandAuthorStore(t) - ctx := context.Background() - - // A record with no author cannot name a commit author, so it reads as a miss. - require.NoError(t, store.RecordCommandAuthor(ctx, "cr-uid", CommandAuthor{})) - _, ok := store.LookupCommandAuthor(ctx, "cr-uid") - require.False(t, ok) -} - -func TestCommandAuthorStore_RecordCarriesCleanupTTL(t *testing.T) { - redisStore, mr := newTestRedisStoreWithRedis(t) - store := redisStore.CommandAuthorStore() - ctx := context.Background() - - require.NoError(t, store.RecordCommandAuthor(ctx, "cr-uid", CommandAuthor{Author: "alice"})) - - // The record carries a fixed cleanup TTL and self-cleans once it elapses (an orphan - // command deleted before its reconcile never blocks Redis). - require.Equal(t, commandAuthorRecordTTL, mr.TTL(store.key("cr-uid"))) - mr.FastForward(commandAuthorRecordTTL + time.Second) - _, ok := store.LookupCommandAuthor(ctx, "cr-uid") - require.False(t, ok) -} - -func TestCommandAuthorStore_KeyReadableFormat(t *testing.T) { - store := newTestCommandAuthorStore(t) - require.Equal(t, "gitops-reverser:author:v1:command:cr-uid", store.key("cr-uid")) -} diff --git a/internal/queue/key_prefix_test.go b/internal/queue/key_prefix_test.go index 185915799..4a8ba07d9 100644 --- a/internal/queue/key_prefix_test.go +++ b/internal/queue/key_prefix_test.go @@ -91,8 +91,6 @@ func TestRedisStore_KeyPrefixReachesEveryKeyFamily(t *testing.T) { stream := store.FactStream(RedisFactStreamConfig{}) require.Equal(t, "cell-a:tenant-7:author:v2:audit:route:default:apps/deployments", stream.streamKey(FactStreamKeyFor("default", gvr.GroupResource()))) - - require.Equal(t, "cell-a:tenant-7:author:v1:command:cr-uid", store.CommandAuthorStore().key("cr-uid")) } func TestRedisStore_DefaultPrefixIsUnchanged(t *testing.T) { @@ -103,7 +101,6 @@ func TestRedisStore_DefaultPrefixIsUnchanged(t *testing.T) { store := newPrefixedRedisStore(t, DefaultKeyPrefix) require.Equal(t, "gitops-reverser:watch:v1:target:gtuid-3:configmaps:cluster:last-rv", store.watchCursorKey("gtuid-3", types.CollectionKeyFor(coreConfigmapsGVR(), ""))) - require.Equal(t, "gitops-reverser:author:v1:command:cr-uid", store.CommandAuthorStore().key("cr-uid")) } // A store built without NewRedisStore — a zero-value struct in a test, a future constructor @@ -116,7 +113,6 @@ func TestRedisStore_ZeroValueStoreStillWritesPrefixedKeys(t *testing.T) { var store RedisStore // keyPrefix == "" require.Equal(t, "gitops-reverser:watch:v1:target:gtuid-3:configmaps:cluster:last-rv", store.watchCursorKey("gtuid-3", types.CollectionKeyFor(coreConfigmapsGVR(), ""))) - require.Equal(t, "gitops-reverser:author:v1:command:cr-uid", store.CommandAuthorStore().key("cr-uid")) require.Equal(t, "gitops-reverser:author:v2:audit:route:default:configmaps", store.FactStream(RedisFactStreamConfig{}).streamKey( FactStreamKeyFor("default", coreConfigmapsGVR().GroupResource()))) diff --git a/internal/queue/redis_store.go b/internal/queue/redis_store.go index fce81530a..2b9e33c4a 100644 --- a/internal/queue/redis_store.go +++ b/internal/queue/redis_store.go @@ -81,14 +81,6 @@ func (s *RedisStore) KeyPrefix() string { return s.keyPrefix } -// CommandAuthorStore builds the command-authorship store on this connection. Wire it -// when the validate-operator-types webhook is enabled; it does not depend on attribution. The -// record lives in the same top-level author domain as audit facts but in the separate -// command subfamily, with its own fixed cleanup TTL. -func (s *RedisStore) CommandAuthorStore() *CommandAuthorStore { - return &CommandAuthorStore{client: s.client, ttl: commandAuthorRecordTTL, keyPrefix: s.keyPrefix} -} - // Ping checks liveness of the underlying Redis/Valkey connection. The readiness gate // uses it so the pod does not report ready before its resume-cursor store is reachable. func (s *RedisStore) Ping(ctx context.Context) error { diff --git a/internal/telemetry/apply_window.go b/internal/telemetry/apply_window.go index 65c3c2e73..92192fb7b 100644 --- a/internal/telemetry/apply_window.go +++ b/internal/telemetry/apply_window.go @@ -22,6 +22,22 @@ import "go.opentelemetry.io/otel/metric" // Registered in registerCounters (exporter.go) with every other counter. var ApplyWindowCollisionsTotal metric.Int64Counter +// ApplyWindowBarriersTotal counts the barriers apply windows took after their operation completed: a +// wait until the watch had delivered every write the source clusters held by then, so the applier's +// own writes are judged inside the window however far behind the watch was. Labelled by +// {gitbranch_*, provider_*, outcome}. outcome is a frozen enum: +// +// met — every stream delivered every such write; the window then closed at the end of its grace. +// timed_out — the window closed at the limit (2 minutes) with a stream still behind: a write of the +// applier's arriving later is mirrored as an edit (apply_echoes_total{outcome="late"}). +// failed — the barrier could not be taken (the source cluster could not be listed, or its +// resourceVersions are not numbers); the window closed after its grace alone. +// +// A steady timed_out or failed rate means the window cannot see every apply it asked for land. +// +// Registered in registerCounters (exporter.go) with every other counter. +var ApplyWindowBarriersTotal metric.Int64Counter + // ApplyEchoesTotal counts the applier's own writes the apply window judged: an apply GitOps Reverser // asked for landing, object by object. Labelled by {gitbranch_*, provider_*, // outcome}. outcome is a frozen enum: diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index df5230ea1..7bde40725 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -149,6 +149,8 @@ var ( // unchanged — a live UPDATE whose sanitized content equals what Git already // holds: the /status-churn case, expected in volume. // bookmark — a watch bookmark, carrying cursor progress and no object. + // cluster_generated — an object the cluster itself writes and Git never holds + // (kube-root-ca.crt), dropped on purpose. // shutdown — the stream was cancelled while the event waited for its author. // Not loss: a restart replays and resyncs the object. // @@ -620,6 +622,7 @@ func registerCounters() error { {"gitopsreverser_apply_claimed_resource_reconciled_total", &ClaimedResourceReconciledTotal}, {"gitopsreverser_apply_window_collisions_total", &ApplyWindowCollisionsTotal}, {"gitopsreverser_apply_echoes_total", &ApplyEchoesTotal}, + {"gitopsreverser_apply_window_barriers_total", &ApplyWindowBarriersTotal}, {"gitopsreverser_apply_stale_snapshots_total", &StaleSnapshotsTotal}, {"gitopsreverser_apply_ownership_conflicts_total", &ApplyOwnershipConflictsTotal}, {"gitopsreverser_receiver_deliveries_total", &ReceiverDeliveriesTotal}, diff --git a/internal/watch/apply_barrier.go b/internal/watch/apply_barrier.go new file mode 100644 index 000000000..f65d504a6 --- /dev/null +++ b/internal/watch/apply_barrier.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "fmt" + "strconv" + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// The watch side of an apply window's barrier (internal/git/apply_barrier.go). A window whose +// operation completed asks, for each GitTarget its applier covers, to hear from every running stream +// once that stream has handled every write its collection held at that moment. The mark is the +// newest resourceVersion among the collection's objects, from a list made now: the position of the +// latest write to anything in it. A stream records how far it has got each time it records its +// cursor, after the event was routed, so the answer it sends from there rides the worker's queue +// behind every event it routed first. +// +// The mark is an object's resourceVersion, not the list's. The list's is the source cluster's +// current position, which a quiet stream only passes on a bookmark, and a bookmark can be a minute +// away. An object's is a write the stream will see, or has seen. +// +// resourceVersions are compared as numbers. The Kubernetes API leaves them opaque, and a source +// whose resourceVersions do not parse as one fails the barrier, so its window closes after its grace +// alone. + +// barrierListPage is the page size of the list a barrier takes of a stream's collection. +const barrierListPage = 500 + +// streamProgress is how far one running target stream has got: the newest resourceVersion it has +// handled, an event or a bookmark, and the barriers waiting for it to get further. +type streamProgress struct { + key targetWatchKey + + mu sync.Mutex + seen uint64 + waiting []progressWaiter +} + +// progressWaiter is one barrier waiting for a stream to reach mark. +type progressWaiter struct { + mark uint64 + deliver func() +} + +// advance records that the stream handled rv, and answers every barrier it has now reached. A +// resourceVersion that is not a number advances nothing. +func (p *streamProgress) advance(rv string) { + n, err := strconv.ParseUint(rv, 10, 64) + if err != nil { + return + } + p.mu.Lock() + p.seen = max(p.seen, n) + var due []func() + kept := p.waiting[:0] + for _, waiter := range p.waiting { + if waiter.mark <= p.seen { + due = append(due, waiter.deliver) + } else { + kept = append(kept, waiter) + } + } + p.waiting = kept + p.mu.Unlock() + for _, deliver := range due { + deliver() + } +} + +// await answers deliver once the stream has handled mark: at once when it already has. +func (p *streamProgress) await(mark uint64, deliver func()) { + p.mu.Lock() + if mark <= p.seen { + p.mu.Unlock() + deliver() + return + } + p.waiting = append(p.waiting, progressWaiter{mark: mark, deliver: deliver}) + p.mu.Unlock() +} + +// trackStreamProgress registers a running stream's progress, replacing a previous stream's for the +// same collection. +func (m *Manager) trackStreamProgress(gitDest types.ResourceReference, key targetWatchKey) *streamProgress { + progress := &streamProgress{key: key} + m.streamProgressMu.Lock() + defer m.streamProgressMu.Unlock() + if m.streamProgress == nil { + m.streamProgress = map[string]map[types.CollectionKey]*streamProgress{} + } + streams := m.streamProgress[gitDest.Key()] + if streams == nil { + streams = map[types.CollectionKey]*streamProgress{} + m.streamProgress[gitDest.Key()] = streams + } + streams[key.Collection()] = progress + return progress +} + +// untrackStreamProgress removes a stopped stream's progress, unless a newer stream replaced it. Its +// waiting barriers are never answered; their windows close at their limit. +func (m *Manager) untrackStreamProgress(gitDest types.ResourceReference, key targetWatchKey, progress *streamProgress) { + m.streamProgressMu.Lock() + defer m.streamProgressMu.Unlock() + streams := m.streamProgress[gitDest.Key()] + if streams[key.Collection()] != progress { + return + } + delete(streams, key.Collection()) + if len(streams) == 0 { + delete(m.streamProgress, gitDest.Key()) + } +} + +// advanceStreamProgress records that the stream of gitDest's collection handled rv. +func (m *Manager) advanceStreamProgress(gitDest types.ResourceReference, key targetWatchKey, rv string) { + if rv == "" { + return + } + m.streamProgressMu.Lock() + progress := m.streamProgress[gitDest.Key()][key.Collection()] + m.streamProgressMu.Unlock() + if progress != nil { + progress.advance(rv) + } +} + +// ApplyBarrier is the watch side of an apply window's barrier (git.ApplyBarrierFunc): it marks every +// running stream of gitDest at the newest write its collection holds now, has each call deliver once +// it has handled that write, and returns how many streams will. It lists the source cluster, so it +// runs off the branch worker's loop. +func (m *Manager) ApplyBarrier(ctx context.Context, gitDest types.ResourceReference, deliver func()) (int, error) { + m.streamProgressMu.Lock() + streams := make([]*streamProgress, 0, len(m.streamProgress[gitDest.Key()])) + for _, progress := range m.streamProgress[gitDest.Key()] { + streams = append(streams, progress) + } + m.streamProgressMu.Unlock() + + clusterID := m.clusterIDForGitTarget(gitDest) + for _, progress := range streams { + mark, err := m.collectionMark(ctx, clusterID, progress.key) + if err != nil { + return 0, fmt.Errorf("mark %s for %s: %w", progress.key.Collection().String(), gitDest.String(), err) + } + progress.await(mark, deliver) + } + return len(streams), nil +} + +// collectionMark is the newest resourceVersion among the objects of a stream's collection: the +// position of the latest write to any of them. An empty collection is zero. +func (m *Manager) collectionMark(ctx context.Context, clusterID string, key targetWatchKey) (uint64, error) { + var mark uint64 + opts := metav1.ListOptions{Limit: barrierListPage} + for { + list, err := m.openTargetList(ctx, clusterID, key, opts) + if err != nil { + return 0, err + } + for i := range list.Items { + rv := list.Items[i].GetResourceVersion() + n, err := strconv.ParseUint(rv, 10, 64) + if err != nil { + return 0, fmt.Errorf("resourceVersion %q of %s is not a number", rv, list.Items[i].GetName()) + } + mark = max(mark, n) + } + if list.GetContinue() == "" { + return mark, nil + } + opts.Continue = list.GetContinue() + } +} diff --git a/internal/watch/apply_barrier_test.go b/internal/watch/apply_barrier_test.go new file mode 100644 index 000000000..d54b46f47 --- /dev/null +++ b/internal/watch/apply_barrier_test.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// barrierList lists objects with the given resourceVersions, two to a page. +func barrierList(rvs ...string) func( + context.Context, schema.GroupVersionResource, string, metav1.ListOptions, +) (*unstructured.UnstructuredList, error) { + return func( + _ context.Context, _ schema.GroupVersionResource, _ string, opts metav1.ListOptions, + ) (*unstructured.UnstructuredList, error) { + start := 0 + if opts.Continue != "" { + start = 2 + } + list := &unstructured.UnstructuredList{} + for i := start; i < len(rvs) && i < start+2; i++ { + item := unstructured.Unstructured{} + item.SetName("cm") + item.SetResourceVersion(rvs[i]) + list.Items = append(list.Items, item) + } + if start == 0 && len(rvs) > 2 { + list.SetContinue("next") + } + return list, nil + } +} + +// A stream answers a barrier once it has handled the newest write its collection held when the +// barrier was taken, an event or a bookmark, and not before. +func TestApplyBarrier_AStreamAnswersOnceItHandledTheNewestWrite(t *testing.T) { + gitDest := types.NewResourceReference("target", "team-a") + key := targetWatchKey{GVR: schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, Namespace: "team-a"} + manager := &Manager{targetWatchList: barrierList("12", "40", "7")} + manager.trackStreamProgress(gitDest, key) + manager.advanceStreamProgress(gitDest, key, "30") + + var delivered atomic.Int32 + streams, err := manager.ApplyBarrier(context.Background(), gitDest, func() { delivered.Add(1) }) + require.NoError(t, err) + assert.Equal(t, 1, streams) + assert.Zero(t, delivered.Load(), "the stream has handled 30, and the newest write, on the second page, is 40") + + manager.advanceStreamProgress(gitDest, key, "39") + assert.Zero(t, delivered.Load()) + manager.advanceStreamProgress(gitDest, key, "41") + assert.Equal(t, int32(1), delivered.Load(), "a bookmark past the mark answers it") + manager.advanceStreamProgress(gitDest, key, "50") + assert.Equal(t, int32(1), delivered.Load(), "once") +} + +// A stream that already handled the newest write answers at once, and a collection with no objects +// asks for nothing. +func TestApplyBarrier_AStreamThatIsCaughtUpAnswersAtOnce(t *testing.T) { + gitDest := types.NewResourceReference("target", "team-a") + key := targetWatchKey{GVR: schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, Namespace: "team-a"} + for _, rvs := range [][]string{{"12", "40"}, {}} { + manager := &Manager{targetWatchList: barrierList(rvs...)} + manager.trackStreamProgress(gitDest, key) + manager.advanceStreamProgress(gitDest, key, "40") + + var delivered atomic.Int32 + streams, err := manager.ApplyBarrier(context.Background(), gitDest, func() { delivered.Add(1) }) + require.NoError(t, err) + assert.Equal(t, 1, streams) + assert.Equal(t, int32(1), delivered.Load(), "%v", rvs) + } +} + +// A source whose resourceVersions are not numbers fails the barrier, and a GitTarget with no +// running stream has nobody to answer. +func TestApplyBarrier_NonNumericResourceVersionsFailAndNoStreamsAnswerNothing(t *testing.T) { + gitDest := types.NewResourceReference("target", "team-a") + key := targetWatchKey{GVR: schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, Namespace: "team-a"} + manager := &Manager{targetWatchList: barrierList("abc")} + + streams, err := manager.ApplyBarrier(context.Background(), gitDest, func() {}) + require.NoError(t, err) + assert.Zero(t, streams, "no stream runs") + + manager.trackStreamProgress(gitDest, key) + _, err = manager.ApplyBarrier(context.Background(), gitDest, func() {}) + require.Error(t, err) +} + +// A stream replaced by a newer one for the same collection keeps the newer one's progress when it +// stops. +func TestApplyBarrier_AStoppedStreamDoesNotRemoveItsReplacement(t *testing.T) { + gitDest := types.NewResourceReference("target", "team-a") + key := targetWatchKey{GVR: schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, Namespace: "team-a"} + manager := &Manager{} + old := manager.trackStreamProgress(gitDest, key) + newer := manager.trackStreamProgress(gitDest, key) + manager.untrackStreamProgress(gitDest, key, old) + assert.Same(t, newer, manager.streamProgress[gitDest.Key()][key.Collection()]) + manager.untrackStreamProgress(gitDest, key, newer) + assert.Empty(t, manager.streamProgress) +} diff --git a/internal/watch/cluster_context.go b/internal/watch/cluster_context.go index 2274a801d..e51edb130 100644 --- a/internal/watch/cluster_context.go +++ b/internal/watch/cluster_context.go @@ -741,19 +741,53 @@ func (m *Manager) ApplierDynamicClient(ctx context.Context, clusterID string) (d if m.sourceClusterDeclared(clusterID) { return m.clusterDynamicClient(ctx, clusterID) } - cfg, _, inCluster, err := m.resolveSourceConfig(ctx, clusterID) + dc, _, err := m.SourceDynamicClient(ctx, clusterID) + return dc, err +} + +// SourceDynamicClient returns a dynamic client for the cluster clusterID names, with that KubeProvider's +// identity, built for this call from the provider as it resolves now, and the version of the credential +// it was built from. Never the cached client of a declared cluster: that one is rebuilt only on the +// credential refresh's own cadence, so a caller that restarts on a new version could still be handed the +// old credential. It is what the CommitRequest server watches a cluster's requests through. +func (m *Manager) SourceDynamicClient(ctx context.Context, clusterID string) (dynamic.Interface, string, error) { + if clusterID == configPlaneClusterID { + return nil, "", errors.New("no cluster named: the config plane is not a KubeProvider") + } + cfg, version, inCluster, err := m.resolveSourceConfig(ctx, clusterID) if err != nil { - return nil, err + return nil, "", err } // Tests inject a fake client for the local cluster without a REST config at all. if inCluster && m.dynamicClient != nil { - return m.dynamicClient, nil + return m.dynamicClient, version, nil } dc, err := dynamic.NewForConfig(cfg) if err != nil { - return nil, fmt.Errorf("build dynamic client for cluster %q: %w", describeCluster(clusterID), err) + return nil, "", fmt.Errorf("build dynamic client for cluster %q: %w", describeCluster(clusterID), err) } - return dc, nil + return dc, version, nil +} + +// SourceDiscovery returns a discovery client for the cluster clusterID names, with that KubeProvider's +// identity, built for this call, and the version of the credential it was built from: what the +// CommitRequest server asks whether the cluster serves the CommitRequest CRD, and whether the credential +// its watch runs on changed. It carries the dial timeout every discovery client here does. +func (m *Manager) SourceDiscovery( + ctx context.Context, clusterID string, +) (discovery.DiscoveryInterface, string, error) { + if clusterID == configPlaneClusterID { + return nil, "", errors.New("no cluster named: the config plane is not a KubeProvider") + } + cfg, version, _, err := m.resolveSourceConfig(ctx, clusterID) + if err != nil { + return nil, "", err + } + dc, err := discovery.NewDiscoveryClientForConfig(discoveryRESTConfig(cfg)) + if err != nil { + return nil, "", fmt.Errorf("build discovery client for cluster %q: %w", describeCluster(clusterID), err) + } + return dc, version, nil } // sourceClusterDeclared reports whether any GitTarget's Declare captured this cluster id. diff --git a/internal/watch/cluster_generated.go b/internal/watch/cluster_generated.go new file mode 100644 index 000000000..42cd174e0 --- /dev/null +++ b/internal/watch/cluster_generated.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// kubeRootCAConfigMap is the ConfigMap kube-controller-manager's root CA publisher writes into every +// namespace. Its content is the cluster's CA bundle: it means nothing outside that one cluster, it +// changes whenever the cluster is rebuilt, and the publisher overwrites any other value at once, so +// a copy in Git can neither be edited nor restored. +const kubeRootCAConfigMap = "kube-root-ca.crt" + +// clusterGenerated reports whether an object is one the cluster itself writes and Git never holds, +// whatever a WatchRule selects. Unlike resource_policy.go, which excludes whole types, this +// excludes single objects of a type that is mirrored. +// +// Both gatherings ask it: the snapshot fold (desiredFromObject) leaves the object out of the +// desired set, and the live route drops its events. An object already in Git is therefore treated +// like any document the cluster does not hold: prune mode "Always" sweeps it on the next resync, +// and "OnEvent" leaves it for a human to delete. +func clusterGenerated(gvr schema.GroupVersionResource, u *unstructured.Unstructured) bool { + if u == nil { + return false + } + return gvr.Group == "" && gvr.Resource == "configmaps" && u.GetName() == kubeRootCAConfigMap +} diff --git a/internal/watch/cluster_generated_test.go b/internal/watch/cluster_generated_test.go new file mode 100644 index 000000000..a87f61181 --- /dev/null +++ b/internal/watch/cluster_generated_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "maps" + "slices" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + + "github.com/ConfigButler/gitops-reverser/internal/reconcile" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +func rootCAConfigMapObject(rv string) *unstructured.Unstructured { + obj := configMapObject(rv) + obj.SetName(kubeRootCAConfigMap) + return obj +} + +func TestClusterGenerated_OnlyTheRootCAConfigMap(t *testing.T) { + secrets := schema.GroupVersionResource{Version: "v1", Resource: "secrets"} + named := func(name string) *unstructured.Unstructured { + obj := configMapObject("1") + obj.SetName(name) + return obj + } + + assert.True(t, clusterGenerated(configmapsGVR, named(kubeRootCAConfigMap))) + assert.False(t, clusterGenerated(configmapsGVR, named("demo")), "any other ConfigMap is mirrored") + assert.False(t, clusterGenerated(secrets, named(kubeRootCAConfigMap)), + "the name alone is not enough: only the ConfigMap is the publisher's") + assert.False(t, clusterGenerated(configmapsGVR, nil)) +} + +// The snapshot leaves it out, so a resync never writes it and, under prune mode "Always", sweeps a +// copy Git already holds: the same as for any object the cluster does not hold. +func TestSnapshotFromList_LeavesOutTheRootCAConfigMap(t *testing.T) { + demo, rootCA := configMapObject("6"), rootCAConfigMapObject("7") + demo.SetUID("demo-uid") + rootCA.SetUID("root-ca-uid") + list := &unstructured.UnstructuredList{Items: []unstructured.Unstructured{*demo, *rootCA}} + + snapshot := snapshotFromList(configmapsGVR, list) + + require.Len(t, snapshot.desired, 1) + assert.Equal(t, "demo", snapshot.desired[0].Resource.Name) + assert.Equal(t, []k8stypes.UID{"demo-uid"}, slices.Collect(maps.Keys(snapshot.baselines)), + "no baseline for an object Git never holds") + assert.Equal(t, 1, snapshot.generated, "counted, so the snapshot can say it left it out") +} + +// Every live event for it is dropped, a DELETED one too: the publisher recreates it at once, and a +// delete mirrored on its own would only make Git's answer depend on which event came last. +func TestRouteLiveTargetWatchEvent_DropsTheRootCAConfigMap(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + gitDest := types.NewResourceReference("my-target", "my-ns") + enqueuer := &recordingEnqueuer{} + stream := reconcile.NewGitTargetEventStream(gitDest.Name, gitDest.Namespace, enqueuer, logr.Discard()) + router := &EventRouter{ + Log: logr.Discard(), + gitTargetStreams: map[string]*reconcile.GitTargetEventStream{gitDest.Key(): stream}, + } + manager := &Manager{EventRouter: router} + key := targetWatchKey{GVR: configmapsGVR, Namespace: "apps"} + + for i, eventType := range []watch.EventType{watch.Added, watch.Modified, watch.Deleted} { + rv, err := manager.routeLiveTargetWatchEvent(context.Background(), logr.Discard(), gitDest, + testStream(key), watch.Event{Type: eventType, Object: rootCAConfigMapObject(string(rune('1' + i)))}) + require.NoError(t, err) + assert.Equal(t, string(rune('1'+i)), rv, "the cursor still moves past it") + } + + assert.Empty(t, enqueuer.events, "nothing reaches the writer") + generated, ok := telemetry.CollectInt64Sum(reader, watchEventsMetric, ingestMatch(watchOutcomeGenerated)) + require.True(t, ok) + assert.Equal(t, int64(3), generated, "each drop is in the ingest census") +} diff --git a/internal/watch/desired_state_change_filter.go b/internal/watch/desired_state_change_filter.go index 68f3b1901..4073058a4 100644 --- a/internal/watch/desired_state_change_filter.go +++ b/internal/watch/desired_state_change_filter.go @@ -125,11 +125,16 @@ type replaySnapshot struct { // owners is the applier that owns each object some applier owns, read from the raw object // before sanitize strips it (git.AppliedByOf). owners map[types.ResourceIdentifier]git.AppliedBy + // generated counts the cluster-generated objects left out (clusterGenerated), for the log. + generated int } // add folds one observed object into the snapshot. An object desiredFromObject excludes (a // Terminating one) gets no baseline either: Git does not hold it. func (r *replaySnapshot) add(gvr schema.GroupVersionResource, u *unstructured.Unstructured) { + if clusterGenerated(gvr, u) { + r.generated++ + } desired, ok := desiredFromObject(gvr, u) if !ok { return diff --git a/internal/watch/event_router.go b/internal/watch/event_router.go index dd26e83e8..2ac5f5313 100644 --- a/internal/watch/event_router.go +++ b/internal/watch/event_router.go @@ -161,6 +161,24 @@ func (r *EventRouter) WithdrawCommitRequest( return git.FinalizeResult{Branch: branch, Phase: phase}, false, nil } +// LookupCommitRequestApply reports where the apply a CommitRequest asked for stands, from the +// branch worker of its GitTarget, and false while that worker knows nothing of it or none runs. +func (r *EventRouter) LookupCommitRequestApply( + ctx context.Context, + attach git.AttachCommitRequest, +) (git.SaveApply, bool, error) { + worker, _, err := r.commitRequestWorker(ctx, attach) + if errors.Is(err, errGitTargetGone) { + return git.SaveApply{Done: true, Applied: true, Reason: git.SaveApplyApplierGone, + Message: "the GitTarget is gone, so its applier is asked for nothing"}, true, nil + } + if err != nil || worker == nil { + return git.SaveApply{}, false, err + } + apply, known := worker.LookupSaveApply(attach.Namespace, attach.Name, attach.UID) + return apply, known, nil +} + // heldBecause is why a request waiting for its push has not reached the remote: the worker's // publication report while it cannot publish, empty otherwise. func heldBecause(worker *git.BranchWorker, phase git.CommitRequestPhase) string { @@ -175,7 +193,9 @@ func heldBecause(worker *git.BranchWorker, phase git.CommitRequestPhase) string // The worker that holds the request comes first, whatever its GitTarget says now: a shared worker // outlives a deleted target, and a retired one finishes its last push after the manager stopped // listing it. Only a request no worker holds falls back to the GitTarget's current worker, which is -// nil while that worker has not started. A GitTarget that is gone is errGitTargetGone. +// nil while that worker has not started. A GitTarget that is gone is errGitTargetGone, and so is one +// that replaced the GitTarget that admitted the request (attach.GitTargetUID): the replacement admitted +// nothing. func (r *EventRouter) commitRequestWorker( ctx context.Context, attach git.AttachCommitRequest, @@ -195,6 +215,10 @@ func (r *EventRouter) commitRequestWorker( return nil, "", fmt.Errorf("get GitTarget %s/%s: %w", attach.GitTargetNamespace, attach.GitTargetName, err) } + if attach.GitTargetUID != "" && string(gitTarget.UID) != attach.GitTargetUID { + return nil, "", fmt.Errorf("%w: %s/%s was replaced", errGitTargetGone, + attach.GitTargetNamespace, attach.GitTargetName) + } worker, exists := r.WorkerManager.WorkerForTarget(&gitTarget) if exists { diff --git a/internal/watch/event_router_test.go b/internal/watch/event_router_test.go index 513d492b7..3aafe2f63 100644 --- a/internal/watch/event_router_test.go +++ b/internal/watch/event_router_test.go @@ -61,6 +61,27 @@ func TestServiceCommitRequest_GitTargetNotFound(t *testing.T) { require.ErrorIs(t, err, errGitTargetGone) } +// A save whose GitTarget is gone needs no apply: Applied=True with ApplierGone. Only a commit the +// branch lost is ever Applied=False. +func TestLookupCommitRequestApply_GitTargetGoneIsApplied(t *testing.T) { + scheme := eventRouterScheme(t) + client := fake.NewClientBuilder().WithScheme(scheme).Build() + workerManager := git.NewWorkerManager( + client, + logr.Discard(), + git.BranchWorkerLimits{}, + types.SensitiveResourcePolicy{}, + ) + router := NewEventRouter(workerManager, nil, client, logr.Discard()) + + apply, known, err := router.LookupCommitRequestApply(context.Background(), saveAttach("missing", "team-a")) + require.NoError(t, err) + require.True(t, known) + assert.True(t, apply.Done) + assert.True(t, apply.Applied, "a save whose GitTarget is gone needs no apply; it is not Applied=False") + assert.Equal(t, git.SaveApplyApplierGone, apply.Reason) +} + func TestGitTargetEventStreamRegistry(t *testing.T) { scheme := eventRouterScheme(t) client := fake.NewClientBuilder().WithScheme(scheme).Build() @@ -293,6 +314,41 @@ func TestGitPathRefusalReason(t *testing.T) { } } +// A GitTarget of the name an attach carries, but with another UID than the GitTarget that admitted +// the request, replaced it and admitted nothing: no worker of it is asked to take the request. +func TestServiceCommitRequest_AReplacedGitTargetIsGone(t *testing.T) { + scheme := eventRouterScheme(t) + gitTarget := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "team-a", UID: "uid-second"}, + Spec: configv1alpha3.GitTargetSpec{ + GitBranchRef: meta.NamespacedObjectReference{Name: gitBranchNameForTest("team-a-provider", "main")}, + }, + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gitTarget, testGitBranchObject()).Build() + workerManager := git.NewWorkerManager( + client, + logr.Discard(), + git.BranchWorkerLimits{}, + types.SensitiveResourcePolicy{}, + ) + router := NewEventRouter(workerManager, nil, client, logr.Discard()) + + attach := saveAttach("team-a-config", "team-a") + attach.GitTargetUID = "uid-first" + _, resolved, err := router.ServiceCommitRequest(context.Background(), attach) + require.ErrorIs(t, err, errGitTargetGone) + assert.False(t, resolved) + + result, resolved, err := router.WithdrawCommitRequest(context.Background(), attach) + require.NoError(t, err) + assert.True(t, resolved, "nothing holds it, and the replacement can never take it") + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + + attach.GitTargetUID = "uid-second" + _, _, err = router.ServiceCommitRequest(context.Background(), attach) + require.NoError(t, err, "the GitTarget that admitted it is asked") +} + // TestServiceCommitRequest_NoWorkerKeepsTheRequestPending pins that a missing worker is not an // outcome. At startup a CommitRequest can reach the controller before its GitTarget's worker exists; // resolving it then would end the save before anything could collect its writes or record its diff --git a/internal/watch/manager.go b/internal/watch/manager.go index fe836ea3c..cc137b63f 100644 --- a/internal/watch/manager.go +++ b/internal/watch/manager.go @@ -172,6 +172,11 @@ type Manager struct { openWatchesMu sync.Mutex openWatches map[openWatchKey]int + // streamProgress is how far each running target stream has got, by GitTarget key and + // collection, for the barriers apply windows take (apply_barrier.go). + streamProgressMu sync.Mutex + streamProgress map[string]map[types.CollectionKey]*streamProgress + // targetWatches is the data plane: one raw watch per (GitTarget, GVR, namespace scope), and // the only source of live object state. Its initial desired set comes from the replay, or from // the buffered LIST fallback when sendInitialEvents is unsupported. diff --git a/internal/watch/resource_policy.go b/internal/watch/resource_policy.go index 837a22820..7df7f2858 100644 --- a/internal/watch/resource_policy.go +++ b/internal/watch/resource_policy.go @@ -23,7 +23,10 @@ func isDefaultResourceExcluded(group, resource string) bool { groupResourceKey("flowcontrol.apiserver.k8s.io", "flowschemas"), groupResourceKey("flowcontrol.apiserver.k8s.io", "prioritylevelconfigurations"), groupResourceKey("batch", "jobs"), - groupResourceKey("batch", "cronjobs"): + groupResourceKey("batch", "cronjobs"), + // A save is a request to GitOps Reverser, never state to mirror. A CommitRequest may sit in a + // namespace a WatchRule mirrors, on a source cluster as on the control cluster. + groupResourceKey("configbutler.ai", "commitrequests"): return true default: return false diff --git a/internal/watch/scope_resolve.go b/internal/watch/scope_resolve.go index 09949a675..0635721ed 100644 --- a/internal/watch/scope_resolve.go +++ b/internal/watch/scope_resolve.go @@ -94,6 +94,9 @@ func desiredFromObject( if u.GetDeletionTimestamp() != nil { return manifestanalyzer.DesiredResource{}, false } + if clusterGenerated(gvr, u) { + return manifestanalyzer.DesiredResource{}, false + } id := types.NewResourceIdentifier(gvr.Group, gvr.Version, gvr.Resource, u.GetNamespace(), u.GetName()) return manifestanalyzer.DesiredResource{Resource: id, Object: sanitize.Sanitize(u)}, true } diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index dc6364bc7..f64a74c19 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -572,6 +572,8 @@ func (m *Manager) runTargetWatch( // sweep, and leave the scope pending all the same. stream.replayed = &atomic.Bool{} stream.changes = newDesiredStateChangeFilter() + progress := m.trackStreamProgress(gitDest, stream.key) + defer m.untrackStreamProgress(gitDest, stream.key, progress) for ctx.Err() == nil { err := m.targetWatchReplayAndStream(ctx, log, gitDest, stream, stream.hasReplayed()) recordWatchSessionEnded(ctx, m.clusterIDForGitTarget(gitDest), stream.key.GVR, sessionEndReason(ctx, err)) @@ -1075,6 +1077,11 @@ func (m *Manager) enqueueReplayResync( // its content is the baseline for those events. A refused or never-completed snapshot never // gets here, and the stream keeps the baselines of what the worker did accept. stream.changes.adopt(snapshot.baselines) + if snapshot.generated > 0 { + log.Info("target snapshot left out cluster-generated objects on purpose; Git never holds them", + "gitDest", gitDest.String(), "gvr", stream.key.GVR.String(), "namespace", stream.key.Namespace, + "name", kubeRootCAConfigMap, "count", snapshot.generated) + } log.V(1).Info("target replay resync enqueued", "gitDest", gitDest.String(), "gvr", stream.key.GVR.String(), "resourceVersion", resourceVersion, "count", len(snapshot.desired)) @@ -1191,6 +1198,13 @@ func (m *Manager) routeLiveTargetWatchEvent( "gvr", stream.key.GVR.String(), "type", string(ev.Type)) return rv, nil } + if clusterGenerated(stream.key.GVR, u) { + recordWatchEvent(ctx, gitDest, stream.key.GVR, watchOutcomeGenerated) + log.V(1).Info("target watch skipped a cluster-generated object on purpose", + "gitDest", gitDest.String(), "gvr", stream.key.GVR.String(), + "namespace", u.GetNamespace(), "name", u.GetName(), "type", string(ev.Type)) + return rv, nil + } op := operationForLiveTargetWatchEvent(ev.Type, u) event := targetWatchGitEvent(stream.key.GVR, u, op) // Stamp the producing collection and stream incarnation before the event leaves the stream. @@ -1462,6 +1476,9 @@ func (m *Manager) recordTargetWatchCursor( key targetWatchKey, rv string, ) error { + // Every event the stream handled, routed or not, and every bookmark passes through here after it + // was routed, which makes it where an apply window's barrier learns how far the stream has got. + m.advanceStreamProgress(gitDest, key, rv) uid := m.resolveGitTargetUID(gitDest) if m.WatchCursorStore == nil || rv == "" || uid == "" { return nil diff --git a/internal/watch/watch_event_metrics.go b/internal/watch/watch_event_metrics.go index 69be3655b..14ec0edbc 100644 --- a/internal/watch/watch_event_metrics.go +++ b/internal/watch/watch_event_metrics.go @@ -30,6 +30,7 @@ const ( watchOutcomeUnchanged = "unchanged" watchOutcomeNotObject = "not_object" watchOutcomeBookmark = "bookmark" + watchOutcomeGenerated = "cluster_generated" watchOutcomeRouteFailed = "route_failed" watchOutcomeShutdown = "shutdown" watchOutcomeStreamError = "stream_error" diff --git a/internal/webhook/admission_allow_handler.go b/internal/webhook/admission_allow_handler.go deleted file mode 100644 index bb5f68936..000000000 --- a/internal/webhook/admission_allow_handler.go +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package webhook - -import ( - "context" - - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -const ( - // ValidateAllPath is the broad observe-all validating admission endpoint. It - // matches every resource and always allows — a test-only capture/observation - // surface today (the e2e SUT wires it; the Helm chart does not), kept as the - // stable extension point for a future cluster-wide policy. Per-our-type handling - // (authorship, config validation) lives on ValidateOperatorTypesPath instead. - ValidateAllPath = "/validate-all" -) - -// AdmissionAllowHandler is a validating admission handler that always allows requests. -type AdmissionAllowHandler struct{} - -// Handle returns an allow response for every admission request. -func (AdmissionAllowHandler) Handle(_ context.Context, _ admission.Request) admission.Response { - return admission.Allowed("allowed") -} diff --git a/internal/webhook/admission_allow_handler_test.go b/internal/webhook/admission_allow_handler_test.go deleted file mode 100644 index 40838c1fc..000000000 --- a/internal/webhook/admission_allow_handler_test.go +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package webhook - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - ctrladmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -func TestAdmissionAllowHandler_AllowsEveryRequest(t *testing.T) { - resp := AdmissionAllowHandler{}.Handle(context.Background(), ctrladmission.Request{}) - - assert.True(t, resp.Allowed) - assert.Equal(t, "allowed", resp.Result.Message) -} diff --git a/internal/webhook/fuzz_test.go b/internal/webhook/fuzz_test.go index 588c0e94a..1ee1a5eee 100644 --- a/internal/webhook/fuzz_test.go +++ b/internal/webhook/fuzz_test.go @@ -10,11 +10,10 @@ import ( ) // FuzzDecodeEventList is the project's dynamic-analysis (fuzzing) gate for the -// network-facing admission/audit surface — the code that decodes untrusted JSON -// posted by the Kubernetes API server. It varies the request body and asserts -// that the whole audit ingress path (decode -> intrinsic accept gate -> record) -// never panics and always answers with a valid HTTP status, and that the sibling -// admission uid probe never panics on the same untrusted bytes. +// network-facing audit surface — the code that decodes untrusted JSON posted by +// the Kubernetes API server. It varies the request body and asserts that the +// whole audit ingress path (decode -> intrinsic accept gate -> record) never +// panics and always answers with a valid HTTP status. // // The handler carries no FactRecorder, so processing is side-effect-free: this // exercises decoding and classification without a live store. Seed inputs are @@ -30,7 +29,6 @@ func FuzzDecodeEventList(f *testing.F) { `{`, // truncated `not json`, ``, - `{"metadata":{"uid":"abc-123"}}`, // admission uid probe shape } for _, s := range seeds { f.Add([]byte(s)) @@ -52,9 +50,5 @@ func FuzzDecodeEventList(f *testing.F) { if w.Code < 100 || w.Code > 599 { t.Fatalf("invalid HTTP status %d for body %q", w.Code, body) } - - // The admission command-authorship path parses the same class of untrusted - // JSON with a lightweight probe; it must also never panic. - _ = commandObjectUID(body) }) } diff --git a/internal/webhook/validate_operator_types_handler.go b/internal/webhook/validate_operator_types_handler.go deleted file mode 100644 index ef09336bb..000000000 --- a/internal/webhook/validate_operator_types_handler.go +++ /dev/null @@ -1,175 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package webhook - -import ( - "context" - "encoding/json" - "time" - - authnv1 "k8s.io/api/authentication/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - "github.com/ConfigButler/gitops-reverser/internal/queue" -) - -// ValidateOperatorTypesPath is the validating admission endpoint scoped to our own -// operator CRDs. Today its one job is command authorship — capturing the submitter of a -// command kind (a CommitRequest) into Redis and always allowing — but it is the intended -// home for per-our-type admission generally (e.g. config validation of WatchRule / -// GitProvider / GitTarget later), which would be added as additional webhook-config -// entries with their own rules and failurePolicy. Distinct from the broad observe-all -// ValidateAllPath. -const ValidateOperatorTypesPath = "/validate-operator-types" - -const ( - // displayNameExtraKey is the user.extra key carrying the OIDC "name" claim, when - // the API server is configured to map it. It mirrors the audit path's extra key - // (internal/queue/audit_event_parsing.go) so a command author and a mirrored- - // resource author resolve the same display name from the same claim. - displayNameExtraKey = "configbutler.ai/claims/display-name" - // emailExtraKey is the user.extra key carrying the OIDC "email" claim. - emailExtraKey = "configbutler.ai/claims/email" -) - -// isCommandKind reports whether gr is one of our own command CRDs whose submitter we -// capture at admission. It is the registry of command kinds: adding one is a single -// case here plus one rule in the webhook config — the handler body does not change. -func isCommandKind(gr metav1.GroupResource) bool { - switch gr { - case metav1.GroupResource{Group: "configbutler.ai", Resource: "commitrequests"}: - return true - // future: case metav1.GroupResource{Group: "configbutler.ai", Resource: ""}: - default: - return false - } -} - -// CommandAuthorRecorder records the authenticated submitter of one command object. -// *queue.CommandAuthorStore satisfies it; the interface keeps the handler unit-testable -// without a live Redis. -type CommandAuthorRecorder interface { - RecordCommandAuthor(ctx context.Context, uid types.UID, author queue.CommandAuthor) error -} - -// ValidateOperatorTypesHandler is the admission handler for our operator CRDs. It dispatches on the -// resource under review and does two unrelated things: -// -// - For a COMMAND kind (a CommitRequest) it captures the authenticated submitter into the -// CommandAuthorStore and always allows — pure observation with a single side effect (a Redis -// upsert), never a rejection, so a user's command never depends on it succeeding (a missed -// capture leaves the request without a claimed actor; see -// docs/spec/commitrequest-admission-authorship.md). -// - For a WatchRule it validates the one cross-object rule admission can usefully give feedback -// on: a second source namespace against a GitTarget that declared its folder namespace-free. -// That one CAN reject, and it is the only thing here that does. -// -// The two branches share an endpoint and nothing else. Validation runs on dry-run (it has no side -// effects, and a server-side dry-run must report the rejection it would get), while the capture -// branch honours NoneOnDryRun. -type ValidateOperatorTypesHandler struct { - Store CommandAuthorRecorder - // Client reads the GitTarget a WatchRule names, and that target's other rules. Nil leaves the - // WatchRule branch evaluating nothing and allowing, which is the same degradation the - // fail-open failurePolicy already provides. - Client client.Client -} - -// Handle dispatches on the resource under review: a WatchRule is validated (and may be rejected), -// and a command CREATE has its {uid → author} recorded before the object persists (the authorship -// invariant, §2) and is then allowed. Every early return on the command path still allows: an -// unrecognised kind, a dry-run, a missing uid, or an unauthenticated request simply records nothing -// and leaves the request without a claimed actor downstream. -func (h *ValidateOperatorTypesHandler) Handle(ctx context.Context, req admission.Request) admission.Response { - log := logf.FromContext(ctx).WithName("validate-operator-types") - - gr := metav1.GroupResource{Group: req.Resource.Group, Resource: req.Resource.Resource} - if isWatchRuleKind(gr) { - return h.validateWatchRuleSourceNamespaces(ctx, req) - } - if !isCommandKind(gr) { - // Belt-and-suspenders; the webhook rules already scope us to the kinds above. - return admission.Allowed("not a kind this endpoint judges") - } - // Dry-run never persists, so the controller will never read this record — and we - // declare sideEffects: NoneOnDryRun, so we must honor it. - if req.DryRun != nil && *req.DryRun { - return admission.Allowed("dry-run: not recorded") - } - - // No store means the webhook is running without a Redis backend (admission is on by - // default, but command-author capture is Redis-backed). Allow without recording; the - // controller then uses the no-actor path (AuthorAttributed=False). - if h.Store == nil { - return admission.Allowed("no author store: not recorded") - } - - // metadata.uid identifies the object (not req.UID, which identifies the review). The - // API server assigns it before validating admission runs, so it is present even for - // a generateName CREATE — no response-body name recovery is ever needed. - uid := commandObjectUID(req.Object.Raw) - if uid == "" { - log.Info("command object has no uid; not recording", "namespace", req.Namespace, "name", req.Name) - return admission.Allowed("no uid: not recorded") - } - - author := queue.CommandAuthor{ - Author: req.UserInfo.Username, // effective (impersonated) user — the apiserver resolved it - DisplayName: firstExtraValue(req.UserInfo.Extra, displayNameExtraKey), - Email: firstExtraValue(req.UserInfo.Extra, emailExtraKey), - RequestedAt: time.Now().UTC().Format(time.RFC3339Nano), - } - if author.Author == "" { - return admission.Allowed("no user to attribute") // anonymous never reaches a persisted CREATE - } - - // Synchronous, best-effort: the write completes before we return, so it is present - // before the object is visible. A failure leaves the request without a claimed actor — - // never block the command. - if err := h.Store.RecordCommandAuthor(ctx, uid, author); err != nil { - log.Error(err, "record command author failed; request will claim no actor", - "resource", gr.String(), "namespace", req.Namespace, "name", req.Name, "uid", uid) - return admission.Allowed("author record failed") - } - // Info (not V(1)) on purpose: command CREATEs are rare, and this line proves the - // webhook was actually invoked and the author captured — the first thing to check - // when a CommitRequest unexpectedly reports AuthorAttributed=False. - log.Info("recorded command author at admission", - "resource", gr.String(), "namespace", req.Namespace, "name", req.Name, - "uid", uid, "author", author.Author) - return admission.Allowed("author recorded") -} - -// commandObjectUID extracts metadata.uid from an admission request's raw object. It -// reads only the field it needs (like the audit handler's metadata probes), so it never -// depends on the command kind being registered in a decoder scheme. -func commandObjectUID(raw []byte) types.UID { - if len(raw) == 0 { - return "" - } - var probe struct { - Metadata struct { - UID types.UID `json:"uid"` - } `json:"metadata"` - } - if err := json.Unmarshal(raw, &probe); err != nil { - return "" - } - return probe.Metadata.UID -} - -// firstExtraValue returns the first value for key in an admission request's -// user.extra map, or "" when the key is absent or carries no values. It mirrors the -// audit path's helper (internal/queue/audit_event_parsing.go) for the same user.extra -// shape carried on an admission request. -func firstExtraValue(extra map[string]authnv1.ExtraValue, key string) string { - values := extra[key] - if len(values) == 0 { - return "" - } - return values[0] -} diff --git a/internal/webhook/validate_operator_types_handler_test.go b/internal/webhook/validate_operator_types_handler_test.go deleted file mode 100644 index 7d184517c..000000000 --- a/internal/webhook/validate_operator_types_handler_test.go +++ /dev/null @@ -1,190 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package webhook - -import ( - "context" - "errors" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - admissionv1 "k8s.io/api/admission/v1" - authnv1 "k8s.io/api/authentication/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrladmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - "github.com/ConfigButler/gitops-reverser/internal/queue" -) - -type recordedAuthor struct { - uid types.UID - author queue.CommandAuthor -} - -// fakeCommandAuthorRecorder captures RecordCommandAuthor calls and can be made to fail. -type fakeCommandAuthorRecorder struct { - calls []recordedAuthor - err error -} - -func (f *fakeCommandAuthorRecorder) RecordCommandAuthor( - _ context.Context, uid types.UID, author queue.CommandAuthor, -) error { - f.calls = append(f.calls, recordedAuthor{uid: uid, author: author}) - return f.err -} - -func commitRequestResource() metav1.GroupVersionResource { - return metav1.GroupVersionResource{Group: "configbutler.ai", Version: "v1alpha3", Resource: "commitrequests"} -} - -// commandReview builds an admission.Request for a command CREATE with the given raw -// object body and submitter identity. -func commandReview( - resource metav1.GroupVersionResource, raw string, user authnv1.UserInfo, dryRun *bool, -) ctrladmission.Request { - return ctrladmission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "review-uid", - Resource: resource, - Name: "save-1", - Namespace: "team-a", - Operation: admissionv1.Create, - UserInfo: user, - DryRun: dryRun, - Object: runtime.RawExtension{Raw: []byte(raw)}, - }, - } -} - -func TestValidateOperatorTypesHandler_RecordsCommandAuthor(t *testing.T) { - rec := &fakeCommandAuthorRecorder{} - h := &ValidateOperatorTypesHandler{Store: rec} - - user := authnv1.UserInfo{ - Username: "alice", - Extra: map[string]authnv1.ExtraValue{ - displayNameExtraKey: {"Alice Liddell"}, - emailExtraKey: {"alice@example.com"}, - }, - } - resp := h.Handle(context.Background(), - commandReview(commitRequestResource(), `{"metadata":{"uid":"cr-uid"}}`, user, nil)) - - assert.True(t, resp.Allowed) - require.Len(t, rec.calls, 1) - assert.Equal(t, types.UID("cr-uid"), rec.calls[0].uid) - got := rec.calls[0].author - assert.Equal(t, "alice", got.Author) - assert.Equal(t, "Alice Liddell", got.DisplayName) - assert.Equal(t, "alice@example.com", got.Email) - assert.NotEmpty(t, got.RequestedAt, "RequestedAt is stamped for lag/debug") -} - -func TestValidateOperatorTypesHandler_NilStoreNoOp(t *testing.T) { - // Admission stays enabled without Redis: no Store, so the handler must allow the - // command without recording (and without a nil dereference). - h := &ValidateOperatorTypesHandler{} - - resp := h.Handle(context.Background(), - commandReview(commitRequestResource(), `{"metadata":{"uid":"cr-uid"}}`, - authnv1.UserInfo{Username: "alice"}, nil)) - - assert.True(t, resp.Allowed, "a missing author store never blocks a command") -} - -func TestValidateOperatorTypesHandler_ServiceAccountSubmitter(t *testing.T) { - rec := &fakeCommandAuthorRecorder{} - h := &ValidateOperatorTypesHandler{Store: rec} - - user := authnv1.UserInfo{Username: "system:serviceaccount:team-a:deployer"} - resp := h.Handle(context.Background(), - commandReview(commitRequestResource(), `{"metadata":{"uid":"sa-uid"}}`, user, nil)) - - assert.True(t, resp.Allowed) - require.Len(t, rec.calls, 1) - assert.Equal(t, "system:serviceaccount:team-a:deployer", rec.calls[0].author.Author) -} - -func TestValidateOperatorTypesHandler_NonCommandKindIsNotRecorded(t *testing.T) { - rec := &fakeCommandAuthorRecorder{} - h := &ValidateOperatorTypesHandler{Store: rec} - - configmaps := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} - resp := h.Handle(context.Background(), - commandReview(configmaps, `{"metadata":{"uid":"cm-uid"}}`, authnv1.UserInfo{Username: "alice"}, nil)) - - assert.True(t, resp.Allowed) - assert.Empty(t, rec.calls, "a non-command kind is never recorded") -} - -func TestValidateOperatorTypesHandler_DryRunIsNotRecorded(t *testing.T) { - rec := &fakeCommandAuthorRecorder{} - h := &ValidateOperatorTypesHandler{Store: rec} - - dryRun := true - resp := h.Handle( - context.Background(), - commandReview( - commitRequestResource(), - `{"metadata":{"uid":"cr-uid"}}`, - authnv1.UserInfo{Username: "alice"}, - &dryRun, - ), - ) - - assert.True(t, resp.Allowed) - assert.Empty(t, rec.calls, "dry-run never persists, so nothing is recorded") -} - -func TestValidateOperatorTypesHandler_MissingUIDIsNotRecorded(t *testing.T) { - rec := &fakeCommandAuthorRecorder{} - h := &ValidateOperatorTypesHandler{Store: rec} - - resp := h.Handle(context.Background(), - commandReview(commitRequestResource(), `{"metadata":{}}`, authnv1.UserInfo{Username: "alice"}, nil)) - - assert.True(t, resp.Allowed) - assert.Empty(t, rec.calls, "without a uid the record cannot be keyed") -} - -func TestValidateOperatorTypesHandler_NoUserIsNotRecorded(t *testing.T) { - rec := &fakeCommandAuthorRecorder{} - h := &ValidateOperatorTypesHandler{Store: rec} - - resp := h.Handle(context.Background(), - commandReview(commitRequestResource(), `{"metadata":{"uid":"cr-uid"}}`, authnv1.UserInfo{}, nil)) - - assert.True(t, resp.Allowed) - assert.Empty(t, rec.calls, "an unauthenticated request names no author") -} - -// A failed Redis write must never reject the command: the handler logs and still -// allows, and the controller degrades to the committer. -func TestValidateOperatorTypesHandler_RecordErrorStillAllows(t *testing.T) { - rec := &fakeCommandAuthorRecorder{err: errors.New("redis down")} - h := &ValidateOperatorTypesHandler{Store: rec} - - resp := h.Handle( - context.Background(), - commandReview( - commitRequestResource(), - `{"metadata":{"uid":"cr-uid"}}`, - authnv1.UserInfo{Username: "alice"}, - nil, - ), - ) - - assert.True(t, resp.Allowed, "a record failure must never block the command") - require.Len(t, rec.calls, 1, "the write was attempted") -} - -func TestCommandObjectUID(t *testing.T) { - assert.Equal(t, types.UID("cr-uid"), commandObjectUID([]byte(`{"metadata":{"uid":"cr-uid"}}`))) - assert.Empty(t, commandObjectUID([]byte(`{"metadata":{}}`))) - assert.Empty(t, commandObjectUID(nil)) - assert.Empty(t, commandObjectUID([]byte("not json"))) -} diff --git a/internal/webhook/watchrule_source_namespace_admission.go b/internal/webhook/watchrule_source_namespace_admission.go deleted file mode 100644 index 3b78f2f4e..000000000 --- a/internal/webhook/watchrule_source_namespace_admission.go +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package webhook - -import ( - "context" - "encoding/json" - "fmt" - "sort" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - v1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" -) - -// isWatchRuleKind reports whether gr is the namespaced WatchRule. ClusterWatchRule is deliberately -// absent: it watches cluster-scoped resources, which have no namespace to bring to a target. -func isWatchRuleKind(gr metav1.GroupResource) bool { - return gr == metav1.GroupResource{Group: "configbutler.ai", Resource: "watchrules"} -} - -// validateWatchRuleSourceNamespaces is the FEEDBACK half of the one-source-namespace rule -// (docs/layout/model.md § "The second guard"): a GitTarget that declared its folder namespace-free -// admits exactly one source namespace, and a rule bringing a second one is rejected here, at the -// moment the mistake is made, instead of leaving a target that refuses every write until someone -// reads its status. -// -// It is not the rule's enforcement. The writer's own precondition is, and it holds whatever this -// returns: this check is one-shot, so it cannot see a serializeNamespace flipped to false after the -// rules were created, the webhook is fail-open (failurePolicy: Ignore), and a cluster can run the -// operator without the admission server at all. Everything that must be true of the bytes is -// decided at the write. See docs/spec/where-validation-lives.md. -// -// Every failure to evaluate ALLOWS, for the same reason: a rejection this handler cannot justify -// is a rejection of the user's object on the strength of an unread GitTarget. -func (h *ValidateOperatorTypesHandler) validateWatchRuleSourceNamespaces( - ctx context.Context, - req admission.Request, -) admission.Response { - log := logf.FromContext(ctx).WithName("watchrule-source-namespaces") - if h.Client == nil { - return admission.Allowed("no client: source namespaces not evaluated") - } - - var rule v1alpha3.WatchRule - if err := json.Unmarshal(req.Object.Raw, &rule); err != nil { - log.Error(err, "could not decode the WatchRule under review", "namespace", req.Namespace, "name", req.Name) - return admission.Allowed("undecodable: source namespaces not evaluated") - } - // A CREATE carries no namespace on the object itself until the API server defaults it. - if rule.Namespace == "" { - rule.Namespace = req.Namespace - } - - var target v1alpha3.GitTarget - targetKey := k8stypes.NamespacedName{Namespace: rule.Namespace, Name: rule.Spec.GitTargetRef.Name} - if err := h.Client.Get(ctx, targetKey, &target); err != nil { - if !apierrors.IsNotFound(err) { - log.Error(err, "could not read the GitTarget this rule names", "gitTarget", targetKey) - } - // A rule naming a target that does not exist yet is ordinary: rules and targets are - // applied in whatever order the manifest happens to list them, and the reconciler holds - // such a rule unready. It is not this check's business. - return admission.Allowed("GitTarget unreadable: source namespaces not evaluated") - } - if target.Spec.SerializeNamespace == nil || *target.Spec.SerializeNamespace { - return admission.Allowed("the target declares no namespace-free folder") - } - - admitted, err := h.admittedSourceNamespaces(ctx, &target, rule.Name) - if err != nil { - log.Error(err, "could not list the sibling WatchRules", "gitTarget", targetKey) - return admission.Allowed("sibling rules unreadable: source namespaces not evaluated") - } - - for _, item := range rule.Spec.Rules { - if item.IsSourceNamespaceWildcard() { - return admission.Denied(fmt.Sprintf( - "GitTarget %s/%s sets spec.serializeNamespace: false, which admits exactly one source "+ - "namespace, and sourceNamespace %q cannot be shown to be one namespace", - target.Namespace, target.Name, v1alpha3.SourceNamespaceWildcard)) - } - admitted[item.EffectiveSourceNamespace(rule.Namespace)] = struct{}{} - } - if len(admitted) <= 1 { - return admission.Allowed("one source namespace") - } - - names := make([]string, 0, len(admitted)) - for ns := range admitted { - names = append(names, ns) - } - sort.Strings(names) - return admission.Denied(fmt.Sprintf( - "GitTarget %s/%s sets spec.serializeNamespace: false, which admits exactly one source namespace, "+ - "but this rule would make %v reach it; the documents it writes carry no metadata.namespace, so "+ - "two namespaces produce one document two objects overwrite in turn. Unset serializeNamespace on "+ - "the target so each document takes the namespace of the root governing it, or point this rule at "+ - "a target of its own", - target.Namespace, target.Name, names)) -} - -// admittedSourceNamespaces is the set already reaching the target, excluding the rule under review -// so an UPDATE is judged on what it would become rather than on what it is. It mirrors -// (internal/git).resolveSourceNamespaces, which computes the same set at the write. -// -// A wildcard among the OTHER rules is not this request's fault, so it adds nothing here: the write -// refuses that target anyway, and rejecting an unrelated edit for it would leave the user unable to -// fix the rule that caused it. -func (h *ValidateOperatorTypesHandler) admittedSourceNamespaces( - ctx context.Context, - target *v1alpha3.GitTarget, - excludeRule string, -) (map[string]struct{}, error) { - var rules v1alpha3.WatchRuleList - if err := h.Client.List(ctx, &rules, client.InNamespace(target.Namespace)); err != nil { - return nil, err - } - admitted := map[string]struct{}{target.Namespace: {}} - for i := range rules.Items { - sibling := &rules.Items[i] - if sibling.Name == excludeRule || sibling.Spec.GitTargetRef.Name != target.Name { - continue - } - for _, item := range sibling.Spec.Rules { - if item.IsSourceNamespaceWildcard() { - continue - } - admitted[item.EffectiveSourceNamespace(sibling.Namespace)] = struct{}{} - } - } - return admitted, nil -} diff --git a/internal/webhook/watchrule_source_namespace_admission_test.go b/internal/webhook/watchrule_source_namespace_admission_test.go deleted file mode 100644 index b0b83f69d..000000000 --- a/internal/webhook/watchrule_source_namespace_admission_test.go +++ /dev/null @@ -1,182 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package webhook - -import ( - "encoding/json" - "testing" - - "github.com/fluxcd/pkg/apis/meta" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - admissionv1 "k8s.io/api/admission/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - ctrladmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - v1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" -) - -// The feedback half of the one-source-namespace rule. Everything asserted here is a rejection or a -// deliberate refusal to judge; the rule itself is enforced at the write, and -// internal/git asserts that half. - -func namespaceFreeTarget(serialize *bool) *v1alpha3.GitTarget { - return &v1alpha3.GitTarget{ - ObjectMeta: metav1.ObjectMeta{Name: "checkout-artifact", Namespace: "shop"}, - Spec: v1alpha3.GitTargetSpec{ - Path: "apps/checkout", - SerializeNamespace: serialize, - }, - } -} - -func watchRuleFor(name, sourceNamespace string) *v1alpha3.WatchRule { - return &v1alpha3.WatchRule{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "shop"}, - Spec: v1alpha3.WatchRuleSpec{ - GitTargetRef: meta.LocalObjectReference{Name: "checkout-artifact"}, - Rules: []v1alpha3.ResourceRule{{ - Resources: []string{"configmaps"}, - SourceNamespace: sourceNamespace, - }}, - }, - } -} - -func watchRuleReview(t *testing.T, rule *v1alpha3.WatchRule, operation admissionv1.Operation) ctrladmission.Request { - t.Helper() - raw, err := json.Marshal(rule) - require.NoError(t, err) - return ctrladmission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Resource: metav1.GroupVersionResource{ - Group: "configbutler.ai", Version: "v1alpha3", Resource: "watchrules"}, - Operation: operation, - Namespace: rule.Namespace, - Name: rule.Name, - Object: runtime.RawExtension{Raw: raw}, - }, - } -} - -func watchRuleHandler(t *testing.T, objects ...client.Object) *ValidateOperatorTypesHandler { - t.Helper() - scheme := runtime.NewScheme() - require.NoError(t, clientgoscheme.AddToScheme(scheme)) - require.NoError(t, v1alpha3.AddToScheme(scheme)) - return &ValidateOperatorTypesHandler{ - Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build(), - } -} - -func TestWatchRuleAdmission_SecondSourceNamespaceIsRejected(t *testing.T) { - no := false - handler := watchRuleHandler(t, namespaceFreeTarget(&no), watchRuleFor("content", "")) - - response := handler.Handle(t.Context(), - watchRuleReview(t, watchRuleFor("content-billing", "billing"), admissionv1.Create)) - - require.False(t, response.Allowed) - assert.Contains(t, response.Result.Message, "admits exactly one source namespace") - assert.Contains(t, response.Result.Message, "[billing shop]") -} - -func TestWatchRuleAdmission_FirstSourceNamespaceIsAdmitted(t *testing.T) { - no := false - handler := watchRuleHandler(t, namespaceFreeTarget(&no)) - - response := handler.Handle(t.Context(), watchRuleReview(t, watchRuleFor("content", ""), admissionv1.Create)) - - assert.True(t, response.Allowed, "the target's own namespace is the one namespace it admits") -} - -// A wildcard is rejected statically: neither reading of "*" can be shown to be one namespace, so -// nothing is enumerated to find out. -func TestWatchRuleAdmission_WildcardIsRejectedWithoutEnumerating(t *testing.T) { - no := false - handler := watchRuleHandler(t, namespaceFreeTarget(&no)) - - response := handler.Handle(t.Context(), - watchRuleReview(t, watchRuleFor("everything", v1alpha3.SourceNamespaceWildcard), admissionv1.Create)) - - require.False(t, response.Allowed) - assert.Contains(t, response.Result.Message, "cannot be shown to be one namespace") -} - -// An UPDATE is judged on what the rule would BECOME. Re-submitting the rule that already holds the -// second namespace must not be rejected for its own existing value, or the only edit that could fix -// it would be blocked by it. -func TestWatchRuleAdmission_UpdateExcludesTheRuleUnderReview(t *testing.T) { - no := false - existing := watchRuleFor("content", "billing") - handler := watchRuleHandler(t, namespaceFreeTarget(&no), existing) - - narrowed := watchRuleFor("content", "shop") - response := handler.Handle(t.Context(), watchRuleReview(t, narrowed, admissionv1.Update)) - - assert.True(t, response.Allowed, "an edit that narrows the rule back to one namespace must land") -} - -func TestWatchRuleAdmission_UnfencedTargetsAreNotJudged(t *testing.T) { - yes := true - for _, tc := range []struct { - name string - serialize *bool - }{ - {"unset infers per document and is never fenced", nil}, - {"true means every document carries its own namespace", &yes}, - } { - t.Run(tc.name, func(t *testing.T) { - handler := watchRuleHandler(t, namespaceFreeTarget(tc.serialize), watchRuleFor("content", "")) - - response := handler.Handle(t.Context(), - watchRuleReview(t, watchRuleFor("content-billing", "billing"), admissionv1.Create)) - - assert.True(t, response.Allowed) - }) - } -} - -// Every way of failing to evaluate ALLOWS. A rejection this handler cannot justify is a rejection -// of the user's object on the strength of a GitTarget nobody read. -func TestWatchRuleAdmission_UnevaluatableRequestsAreAllowed(t *testing.T) { - no := false - - t.Run("no client", func(t *testing.T) { - handler := &ValidateOperatorTypesHandler{} - response := handler.Handle(t.Context(), - watchRuleReview(t, watchRuleFor("content-billing", "billing"), admissionv1.Create)) - assert.True(t, response.Allowed) - }) - - t.Run("the GitTarget does not exist yet", func(t *testing.T) { - handler := watchRuleHandler(t) - response := handler.Handle(t.Context(), - watchRuleReview(t, watchRuleFor("content-billing", "billing"), admissionv1.Create)) - assert.True(t, response.Allowed, "rules and targets are applied in whatever order the manifest lists them") - }) - - t.Run("an undecodable object", func(t *testing.T) { - handler := watchRuleHandler(t, namespaceFreeTarget(&no)) - review := watchRuleReview(t, watchRuleFor("content", ""), admissionv1.Create) - review.Object.Raw = []byte("{not json") - assert.True(t, handler.Handle(t.Context(), review).Allowed) - }) -} - -// A rule pointing at a different GitTarget contributes nothing, so a target fenced to one namespace -// is not rejected because of somebody else's rule. -func TestWatchRuleAdmission_OtherTargetsRulesAreNotCounted(t *testing.T) { - no := false - elsewhere := watchRuleFor("elsewhere", "billing") - elsewhere.Spec.GitTargetRef.Name = "other-artifact" - handler := watchRuleHandler(t, namespaceFreeTarget(&no), elsewhere) - - response := handler.Handle(t.Context(), watchRuleReview(t, watchRuleFor("content", ""), admissionv1.Create)) - - assert.True(t, response.Allowed) -} diff --git a/test/e2e/applier_claim.go b/test/e2e/applier_claim.go index 42c22ebbc..f2d0cc2da 100644 --- a/test/e2e/applier_claim.go +++ b/test/e2e/applier_claim.go @@ -20,6 +20,10 @@ const ( // createFluxClaim creates the FluxClaim name in namespace, holding the Kustomization // flux-system/ in the operator's own cluster (the install's "local" KubeProvider). +// +// Its spec.catchUpAfter is "0s", not the "1m" default: the specs count the applies an edit or a +// push asks for, and a converge a minute after the worker's own push would land inside their holds. +// The spec that tests the converge sets one (setFluxClaimCatchUpAfter). func createFluxClaim(namespace, name, kustomization string) { GinkgoHelper() manifest := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 @@ -32,11 +36,20 @@ spec: kustomizationRef: namespace: flux-system name: %s + catchUpAfter: 0s `, name, namespace, localKubeProviderRef(), kustomization) _, err := kubectlRunWithStdin(namespace, manifest, "apply", "-f", "-") Expect(err).NotTo(HaveOccurred(), "failed to create FluxClaim %s/%s", namespace, name) } +// setFluxClaimCatchUpAfter sets the FluxClaim's spec.catchUpAfter. +func setFluxClaimCatchUpAfter(namespace, name, interval string) { + GinkgoHelper() + _, err := kubectlRunInNamespace(namespace, "patch", fluxClaimResource, name, "--type=merge", "-p", + fmt.Sprintf(`{"spec":{"catchUpAfter":%q}}`, interval)) + Expect(err).NotTo(HaveOccurred(), "failed to set the catchUpAfter of FluxClaim %s/%s", namespace, name) +} + // createArgoCDClaim creates the ArgoCDClaim name in namespace, holding the Application // / in the operator's own cluster. func createArgoCDClaim(namespace, name, appNamespace, application string) { diff --git a/test/e2e/author_mode_test.go b/test/e2e/author_mode_test.go index 89ddf15e3..70c41067e 100644 --- a/test/e2e/author_mode_test.go +++ b/test/e2e/author_mode_test.go @@ -16,7 +16,7 @@ import "testing" // configured-author row below therefore turns attribution off explicitly. func TestConfiguredAuthorModeFromArgs_AllScenarios(t *testing.T) { // The real jsonpath output shape: a bracketed, quoted, comma-separated list. - const live = `["--metrics-bind-address=:8443","--admission-webhook",` + + const live = `["--metrics-bind-address=:8443",` + `"--redis-addr=valkey.valkey-e2e.svc.cluster.local:6379","--redis-insecure"]` for _, tc := range []struct { diff --git a/test/e2e/bi_directional_echoes_e2e_test.go b/test/e2e/bi_directional_echoes_e2e_test.go index c65c1f639..927839be6 100644 --- a/test/e2e/bi_directional_echoes_e2e_test.go +++ b/test/e2e/bi_directional_echoes_e2e_test.go @@ -93,6 +93,16 @@ type echoRun struct { *timingPipeline applier echoApplier + // kubectl runs kubectl in testNs on the cluster the GitTarget mirrors; nil is the operator's own. + kubectl func(args ...string) (string, error) +} + +// liveKubectl runs kubectl in testNs on the cluster the GitTarget mirrors. +func (r *echoRun) liveKubectl(args ...string) (string, error) { + if r.kubectl != nil { + return r.kubectl(args...) + } + return kubectlRunInNamespace(r.testNs, args...) } // cmFile is where the GitTarget writes the ConfigMap name. @@ -124,7 +134,7 @@ func (r *echoRun) createThroughAPI(name, value string) { // editThroughAPI sets data.value of the ConfigMap name through the Kubernetes API. func (r *echoRun) editThroughAPI(name, value string) { GinkgoHelper() - _, err := kubectlRunInNamespace(r.testNs, "patch", "configmap", name, "--type=merge", "-p", + _, err := r.liveKubectl("patch", "configmap", name, "--type=merge", "-p", fmt.Sprintf(`{"data":{"value":%q}}`, value)) Expect(err).NotTo(HaveOccurred(), "failed to edit ConfigMap %s through the API", name) } @@ -144,7 +154,7 @@ func (r *echoRun) committedValue(g Gomega, name string) (string, bool) { // liveValue is data.value of the live ConfigMap name, and whether it exists. func (r *echoRun) liveValue(g Gomega, name string) (string, bool) { - out, err := kubectlRunInNamespace(r.testNs, "get", "configmap", name, "--ignore-not-found", + out, err := r.liveKubectl("get", "configmap", name, "--ignore-not-found", "-o", "jsonpath={.metadata.name}|{.data.value}") g.Expect(err).NotTo(HaveOccurred()) gotName, value, _ := strings.Cut(strings.TrimSpace(out), "|") @@ -220,6 +230,27 @@ func (r *echoRun) eventuallyPruned(name string) { }, echoHold, 2*time.Second).Should(Succeed()) } +// editDuringApply pushes a Git-side change to one object and edits another through the API while +// that apply runs: the apply reverts the edit on the cluster, and the window withholds that write. +// The edit is in Git at once, and on the cluster again after the Repair (E5). +func (r *echoRun) editDuringApply() { + GinkgoHelper() + r.pushFromOutside("during", "d0") + r.eventuallyEverywhere("during", "d0", echoApplyBudget) + + By("pushing a Git-side change to another object, and editing this one while that apply runs") + before := r.applier.operation(Default).Token + r.pushFromOutside("during-other", "o1") + Eventually(func(g Gomega) { + g.Expect(r.applier.operation(g).Token).NotTo(Equal(before), "the apply is asked for") + }, echoApplyBudget, 250*time.Millisecond).Should(Succeed()) + r.editThroughAPI("during", "d-edit") + + r.eventuallyEverywhere("during", "d-edit", echoApplyBudget) + r.eventuallyEverywhere("during-other", "o1", echoApplyBudget) + r.consistentlyEverywhere("during", "d-edit") +} + // echoScenarios are the scenarios that run the same for both appliers, against run. func echoScenarios(run func() *echoRun) { It("adopts an object the API created, so a Git-side delete prunes it (E2)", func() { @@ -250,21 +281,7 @@ func echoScenarios(run func() *echoRun) { }) It("keeps an edit made during an apply, in Git at once and on the cluster after the Repair (E5)", func() { - r := run() - r.pushFromOutside("during", "d0") - r.eventuallyEverywhere("during", "d0", echoApplyBudget) - - By("pushing a Git-side change to another object, and editing this one while that apply runs") - before := r.applier.operation(Default).Token - r.pushFromOutside("during-other", "o1") - Eventually(func(g Gomega) { - g.Expect(r.applier.operation(g).Token).NotTo(Equal(before), "the apply is asked for") - }, echoApplyBudget, 250*time.Millisecond).Should(Succeed()) - r.editThroughAPI("during", "d-edit") - - r.eventuallyEverywhere("during", "d-edit", echoApplyBudget) - r.eventuallyEverywhere("during-other", "o1", echoApplyBudget) - r.consistentlyEverywhere("during", "d-edit") + run().editDuringApply() }) It("applies a Git-side push while edits keep arriving, and keeps every edit (E6)", func() { @@ -380,6 +397,7 @@ func echoScenarios(run func() *echoRun) { By("saving an API edit with a CommitRequest while a Git-side push changes the same object") name := applyCommitRequestWithGenerateName(r.testNs, "echo-save-", r.targetName, "e2e: save the edit") + waitForCommitRequestClaimed(r.testNs, name) r.editThroughAPI("saved", "s-save") r.pushFromOutside("saved", "s-git") diff --git a/test/e2e/bi_directional_timing_e2e_test.go b/test/e2e/bi_directional_timing_e2e_test.go index e85f35585..c4604cbaa 100644 --- a/test/e2e/bi_directional_timing_e2e_test.go +++ b/test/e2e/bi_directional_timing_e2e_test.go @@ -129,21 +129,7 @@ func (p *timingPipeline) start() { p.hookID = registerOperatorReceiver(p.testNs, p.receiverName, p.repo.RepoName) createGitTarget(p.targetName, p.testNs, p.providerName, p.gitPath, "main") - _, err = kubectlRunWithStdin(p.testNs, fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 -kind: WatchRule -metadata: - name: %s - namespace: %s -spec: - gitTargetRef: - name: %s - rules: - - resources: ["configmaps"] - objectSelector: - matchLabels: - %s: "true" -`, p.ruleName, p.testNs, p.targetName, timingLabelKey), "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred(), "failed to apply the WatchRule") + p.applyWatchRule() verifyResourceStatus("gittarget", p.targetName, p.testNs, "True", "Succeeded", "") waitForStreamsRunning(p.targetName, p.testNs) @@ -163,6 +149,26 @@ data: }, timingCommitBudget, timingPoll).Should(Succeed(), "the ConfigMap must be in Git before an applier applies it") } +// applyWatchRule applies the WatchRule that mirrors the labeled ConfigMaps of testNs to the GitTarget. +func (p *timingPipeline) applyWatchRule() { + GinkgoHelper() + _, err := kubectlRunWithStdin(p.testNs, fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: WatchRule +metadata: + name: %s + namespace: %s +spec: + gitTargetRef: + name: %s + rules: + - resources: ["configmaps"] + objectSelector: + matchLabels: + %s: "true" +`, p.ruleName, p.testNs, p.targetName, timingLabelKey), "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to apply the WatchRule") +} + func (p *timingPipeline) cleanup() { deleteRepoHook(p.repo.RepoName, p.hookID) cleanupWatchRule(p.ruleName, p.testNs) diff --git a/test/e2e/cluster/audit/policy.yaml b/test/e2e/cluster/audit/policy.yaml index bfc033c8b..2aa32aca1 100644 --- a/test/e2e/cluster/audit/policy.yaml +++ b/test/e2e/cluster/audit/policy.yaml @@ -53,7 +53,7 @@ rules: - deletecollection # No resources listed = Matches everything else (including CRDs). # This also captures `commitrequests.configbutler.ai` create events, which - # drive the "finalize the open commit window now" path. If this catch-all - # is ever narrowed, keep at least Metadata-level capture for - # `commitrequests` create events — the consumer reads the object body from - # the apiserver, so Metadata is sufficient. + # name a CommitRequest's author, as a write's event names its author. If this + # catch-all is ever narrowed, keep at least Metadata-level capture for + # `commitrequests` create events — the author is joined by uid and + # resourceVersion, which Metadata carries. diff --git a/test/e2e/commit_request_e2e_test.go b/test/e2e/commit_request_e2e_test.go index 1f6a7bbfa..b8945f06c 100644 --- a/test/e2e/commit_request_e2e_test.go +++ b/test/e2e/commit_request_e2e_test.go @@ -26,7 +26,7 @@ import ( // namespace. The HEAD/SHA assertions below therefore read back only this spec's // own commit; concurrent audit traffic for other GitTargets lands in other // repos and cannot move this repo's HEAD. See docs/spec/e2e-serial-registry.md. -var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Ordered, func() { +var _ = Describe("Save Request", Label("commit-request", "audit-consumer"), Ordered, func() { var ( testNs string repo *RepoArtifacts @@ -54,7 +54,7 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or seed := GinkgoRandomSeed() deployName := fmt.Sprintf("commit-request-deploy-%d", seed) commitRequestName := fmt.Sprintf("commit-request-save-%d", seed) - message := fmt.Sprintf("save: commit request from e2e seed %d", seed) + message := fmt.Sprintf("save: save request from e2e seed %d", seed) By("creating a Deployment to open a commit window") applyScaleTestDeployment(testNs, deployName, 0) @@ -112,6 +112,20 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or recentCommitDiagnostics(repo.CheckoutDir, basePath)) }, 2*time.Minute, 3*time.Second).Should(Succeed()) + if !configuredAuthorModeEnabled() { + // The round trip admission used to stand in for: the request's author comes from its own + // audit event, and it is the author of the edit's window, which it saved. + By("verifying the CommitRequest's author came from audit and authored the commit") + Eventually(func(g Gomega) { + author := commitRequestField(g, testNs, commitRequestName, "{.status.author}") + g.Expect(author).NotTo(BeEmpty(), "the request's author is found from its audit event") + commitAuthor, logErr := gitRun(repo.CheckoutDir, "log", "-1", "--pretty=%an") + g.Expect(logErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(commitAuthor)).To(Equal(author), + "the request saved its own author's window") + }, 30*time.Second, 2*time.Second).Should(Succeed()) + } + // The wiring check for the metric instruments, made here because this is where the evidence // is: the commit above is verified in Git, so a missing series is a broken exporter rather // than a spec that did not run. Both queries are scoped to this suite's own GitTarget, so @@ -216,21 +230,15 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or _, _ = kubectlRunInNamespace(testNs, "delete", "commitrequest", commitRequestName, "--ignore-not-found=true") }) - // generateName authorship is now captured at admission, not from the audit - // response body: the API server assigns metadata.name and metadata.uid before - // validating admission runs, so the validate-operator-types webhook records the - // submitter keyed by uid with no response-body name recovery (the old audit - // generateName headache is gone, see - // docs/spec/commitrequest-admission-authorship.md). This spec proves a - // generateName CommitRequest still finalizes and becomes Ready. It is skipped in - // configured-author mode, where the edit's window is committer-authored and the - // named admission author would not match it end to end. + // A generateName CommitRequest is attributed from its audit fact, keyed by the uid and + // resourceVersion the API server assigned, so no name recovery from the response body is needed. + // This spec proves a generateName CommitRequest still finalizes and becomes Ready. It("finalizes a CommitRequest created with metadata.generateName", func() { basePath := "e2e/commit-request-test" seed := GinkgoRandomSeed() deployName := fmt.Sprintf("commit-request-gen-deploy-%d", seed) commitRequestPrefix := fmt.Sprintf("commit-request-gen-%d-", seed) - message := fmt.Sprintf("save: generateName commit request from e2e seed %d", seed) + message := fmt.Sprintf("save: generateName save request from e2e seed %d", seed) By("creating a Deployment to open a commit window") applyScaleTestDeployment(testNs, deployName, 0) @@ -278,7 +286,7 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or // Deployment WatchRule) makes the one-commit assertion unambiguous: main does not // exist until the bundle is finalized, so the bundle commit is the only commit on // main. -var _ = Describe("Commit Request Bundle (UC2)", Label("commit-request", "audit-consumer"), Ordered, func() { +var _ = Describe("Save Request Bundle (UC2)", Label("commit-request", "audit-consumer"), Ordered, func() { var ( testNs string repo *RepoArtifacts @@ -426,9 +434,9 @@ spec: // applyCommitRequestWithGenerateName creates a CommitRequest using // metadata.generateName and returns the server-allocated name. The spec creates its work and this -// CommitRequest back-to-back: authorship is settled synchronously at admission (no controller-side -// wait), so a zero wait would race the work's watch event and could resolve NoWindow before the -// window opens. Waiting to attach is the documented mechanism for a CommitRequest issued +// CommitRequest back-to-back: the request's author and its work's arrive through the same audit join, +// so a zero wait would race the work's watch event and could resolve NoWindow before the window +// opens. Waiting to attach is the documented mechanism for a CommitRequest issued // concurrently with its work (UC2). // // The wait is deliberately tight: it covers audit delivery (--audit-webhook-batch-max-wait=1s) and @@ -480,6 +488,18 @@ spec: Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to apply CommitRequest %s/%s", namespace, name)) } +// waitForCommitRequestClaimed waits until the request is claimed: Ready=Unknown, written once its own +// audit event named its author and before it is attached. A save made before its edit waits for this +// on a GitTarget whose windows close on the first silence (e2eCommitWindow): the request is +// registered only after its author is found, and its edit, attributed the same way, could otherwise +// reach the worker first and commit alone. +func waitForCommitRequestClaimed(namespace, name string) { + GinkgoHelper() + Eventually(func(g Gomega) { + g.Expect(commitRequestCondition(g, namespace, name, "Ready")).To(Equal("Unknown")) + }, time.Minute, 250*time.Millisecond).Should(Succeed(), "CommitRequest %s/%s is claimed", namespace, name) +} + // commitRequestField reads a jsonpath field off a CommitRequest object. func commitRequestField(g Gomega, namespace, name, jsonPath string) string { out, err := kubectlRunInNamespace(namespace, "get", "commitrequest", name, @@ -603,10 +623,9 @@ func setupWindowedDeploymentPipeline(name, basePath, commitWindow string) window By(fmt.Sprintf("creating GitTarget with spec.commit.window=%s", commitWindow)) createValidatedGitTargetWithCommitWindow(p.gitTarget, p.ns, p.gitProvider, basePath, commitWindow) - // Watch Deployments, not ConfigMaps: a fresh namespace contains NO Deployments, whereas every - // namespace is pre-populated with a kube-root-ca.crt ConfigMap that a configmaps WatchRule - // would match — its initial reconcile would establish main before the spec's own edit, hiding - // the pure "branch not even created until something is finalized" behaviour these suites test. + // Watch Deployments: a fresh namespace contains NO Deployments, so the initial reconcile + // commits nothing and cannot establish main before the spec's own edit, which would hide the + // pure "branch not even created until something is finalized" behaviour these suites test. applyDeploymentWatchRule(p.ns, p.watchRule, p.gitTarget) verifyResourceStatus("watchrule", p.watchRule, p.ns, "True", "Succeeded", "") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a9031f130..21a81de4a 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -143,7 +143,7 @@ var commitRequestOutcomes = []string{ // DIAGNOSTIC, not a gate. A suite-wide "none failed" assertion would first have to establish that // the last save had been scraped, and until that is handled a green result would mean less than it // looks. The wiring claim this counter has to support is made where the evidence is local: the -// Commit Request spec asserts its own save appears, scoped to its own GitTarget. +// Save Request spec asserts its own save appears, scoped to its own GitTarget. // // The breakdown is the point. `committed` is what the happy path looks like and the only value // these specs produce: `window_mismatch` needs two authors contending for one branch's window, @@ -157,7 +157,7 @@ func reportCommitRequestOutcomes() { n, err := queryPrometheus(fmt.Sprintf( `sum(max_over_time(gitopsreverser_commit_requests_total{outcome=%q}[2h])) or vector(0)`, oc)) if err == nil { - _, _ = fmt.Fprintf(GinkgoWriter, " commit request outcome %-16s = %.0f\n", oc, n) + _, _ = fmt.Fprintf(GinkgoWriter, " save request outcome %-16s = %.0f\n", oc, n) } } } diff --git a/test/e2e/flux_applier_e2e_test.go b/test/e2e/flux_applier_e2e_test.go index 7534ca928..8b3a2d06b 100644 --- a/test/e2e/flux_applier_e2e_test.go +++ b/test/e2e/flux_applier_e2e_test.go @@ -293,6 +293,115 @@ data: g.Expect(kustomizationSuspended(g, fluxName)).To(BeTrue(), "suspended again") }, 2*time.Minute, 2*time.Second).Should(Succeed()) verifyResourceCondition(fluxClaimResource, fluxName, testNs, "InControl", "True", "Suspended", "") + + // The namespace goes when the spec ends. A ConfigMap left in Git for it would fail every later + // apply of the folder, so it leaves Git first, and the folder applies without it. + By("taking the ConfigMap out of Git again, before its namespace goes") + removeFileFromOutside(repo, fluxApplierFolder+"/later.yaml", "e2e: another writer removes the ConfigMap") + Eventually(func(g Gomega) { + head := remoteHeadSHA(g, repo.CheckoutDir) + g.Expect(fluxLastAppliedRevision(g, fluxName)).To(ContainSubstring(head)) + g.Expect(readClaimOperation(g, testNs, fluxClaimResource, fluxName).Phase).To(Equal("Succeeded")) + }, 3*time.Minute, 2*time.Second).Should(Succeed()) + }) + + // A push of GitOps Reverser's own asks no apply at once, but the Kustomization's own state follows + // the branch at the applier claim's spec.catchUpAfter: its lastAppliedRevision and its + // GitRepository's artifact reach the tip a mirrored API edit made, with no push by anyone else. + // + // The timed edit is an UPDATE of an object the Kustomization already applied. A create would not + // do: it is adopted at once (a CatchUp that stamps Flux's owner labels), with no interval at all. + It("brings the applied revision to its own pushes' tip at the catchUpAfter", func() { + const name = "applier-converge-edit" + + By("creating a ConfigMap, and waiting until its adopt has applied the tip") + applyConfigMap(testNs, name) + var adoptedHead string + var adopted claimOperation + Eventually(func(g Gomega) { + g.Expect(runGitIn(repo.CheckoutDir, "fetch", "origin", "main")).To(Succeed()) + g.Expect(gitShowFiles(repo.CheckoutDir)).To(ContainSubstring(name)) + adoptedHead = remoteHeadSHA(g, repo.CheckoutDir) + g.Expect(fluxLastAppliedRevision(g, fluxName)).To(ContainSubstring(adoptedHead)) + adopted = readClaimOperation(g, testNs, fluxClaimResource, fluxName) + g.Expect(adopted.Phase).To(Equal("Succeeded")) + }, 3*time.Minute, 2*time.Second).Should(Succeed()) + + // No reset afterwards: the next spec deletes the applier claim. + setFluxClaimCatchUpAfter(testNs, fluxName, "30s") + + By("editing it through the API: the branch moves by GitOps Reverser's own push") + _, err := kubectlRunInNamespace(testNs, "patch", "configmap", name, "--type=merge", "-p", + `{"data":{"test-key":"converged"}}`) + Expect(err).NotTo(HaveOccurred()) + var editedHead string + Eventually(func(g Gomega) { + editedHead = remoteHeadSHA(g, repo.CheckoutDir) + g.Expect(editedHead).NotTo(Equal(adoptedHead), "the edit is pushed") + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("no apply at once: the cluster already holds the edit") + Consistently(func(g Gomega) { + g.Expect(readClaimOperation(g, testNs, fluxClaimResource, fluxName).Token).To(Equal(adopted.Token)) + g.Expect(fluxLastAppliedRevision(g, fluxName)).NotTo(ContainSubstring(editedHead)) + }, 10*time.Second, 2*time.Second).Should(Succeed()) + + By("once the interval has passed, a CatchUp applies the tip, and the Kustomization is held again") + Eventually(func(g Gomega) { + g.Expect(fluxLastAppliedRevision(g, fluxName)).To(ContainSubstring(editedHead)) + g.Expect(fluxArtifactRevision(g, fluxName)).To(ContainSubstring(editedHead)) + done := readClaimOperation(g, testNs, fluxClaimResource, fluxName) + g.Expect(done.Token).NotTo(Equal(adopted.Token)) + g.Expect(done.Reason).To(Equal("CatchUp")) + g.Expect(done.Phase).To(Equal("Succeeded")) + g.Expect(kustomizationSuspended(g, fluxName)).To(BeTrue()) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + }) + + // A save that asks to catch up is applied as soon as it is pushed, with catchUpAfter off, and its + // CommitRequest reports the apply: Applied=True with the revision Flux applied. + It("applies a save that asks to catch up, and reports it on the CommitRequest", func() { + const configMap, request = "applier-converge-edit", "applier-save-catch-up" + setFluxClaimCatchUpAfter(testNs, fluxName, "0s") + + By("saving an API edit with catchUp: AfterPush") + _, err := kubectlRunWithStdin(testNs, fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: CommitRequest +metadata: + name: %s + namespace: %s +spec: + gitTargetRef: + name: %s + message: "e2e: save and apply" + catchUp: AfterPush + window: + attachTimeout: "4s" + maxDuration: "4s" +`, request, testNs, destName), "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred()) + waitForCommitRequestClaimed(testNs, request) + _, err = kubectlRunInNamespace(testNs, "patch", "configmap", configMap, "--type=merge", "-p", + `{"data":{"test-key":"saved"}}`) + Expect(err).NotTo(HaveOccurred()) + + By("the save is pushed, and Ready says only that") + var commit string + Eventually(func(g Gomega) { + g.Expect(commitRequestCondition(g, testNs, request, "Ready")).To(Equal("True")) + g.Expect(commitRequestField(g, testNs, request, "{.status.conditions[?(@.type==\"Ready\")].reason}")). + To(Equal(commitRequestReasonCommitted)) + commit = commitRequestField(g, testNs, request, "{.status.commit}") + g.Expect(commit).NotTo(BeEmpty()) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("the applier applies it at once, and Applied reports the revision") + Eventually(func(g Gomega) { + g.Expect(commitRequestCondition(g, testNs, request, "Applied")).To(Equal("True")) + g.Expect(commitRequestField(g, testNs, request, "{.status.appliedRevision}")).To(ContainSubstring(commit)) + g.Expect(fluxLastAppliedRevision(g, fluxName)).To(ContainSubstring(commit)) + g.Expect(readClaimOperation(g, testNs, fluxClaimResource, fluxName).Reason).To(Equal("CatchUp")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) }) // Deleting the applier claim hands the Kustomization back: its finalizer waits until nothing is @@ -418,6 +527,14 @@ func fluxLastAppliedRevision(g Gomega, name string) string { return strings.TrimSpace(out) } +// fluxArtifactRevision is the revision of the GitRepository name's artifact. +func fluxArtifactRevision(g Gomega, name string) string { + out, err := kubectlRunInNamespace("flux-system", "get", "gitrepository", name, + "-o", "jsonpath={.status.artifact.revision}") + g.Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(out) +} + // gitTargetEventMessages is every message of the Events with reason recorded on a GitTarget. func gitTargetEventMessages(g Gomega, testNs, gitTarget, reason string) string { out, err := kubectlRunInNamespace(testNs, "get", "events", "--field-selector", diff --git a/test/e2e/gittarget_overlap_e2e_test.go b/test/e2e/gittarget_overlap_e2e_test.go index 358e7874e..36f66d4bc 100644 --- a/test/e2e/gittarget_overlap_e2e_test.go +++ b/test/e2e/gittarget_overlap_e2e_test.go @@ -16,8 +16,7 @@ import ( // GitTarget path may be equal to, an ancestor of, or a descendant of another's. // Sibling folders are fine; nesting is rejected at the Validated gate so every // materialized folder has exactly one owner. The reject path surfaces as a -// reconcile-time status condition (Ready=False / ValidationFailed), not through -// the current allow-only validating admission webhook. +// reconcile-time status condition (Ready=False / ValidationFailed). var _ = Describe("Manager GitTarget Overlap Guard", Label("manager"), Ordered, func() { const providerName = "gitprovider-overlap" diff --git a/test/e2e/inplace_edit_e2e_test.go b/test/e2e/inplace_edit_e2e_test.go index 6edf5dbb2..50c28aa93 100644 --- a/test/e2e/inplace_edit_e2e_test.go +++ b/test/e2e/inplace_edit_e2e_test.go @@ -272,16 +272,10 @@ var _ = Describe( // holds every line the human wrote, in order, and that anything added to it is a `resources:` // entry. // -// It replaces a byte-equality assertion, which is no longer the right question. This namespace -// holds a ConfigMap nobody in the test created — the cluster's own `kube-root-ca.crt` — and the -// WatchRule selects every ConfigMap, so the operator has a watched resource with no document in -// Git. Placement gives it a file beside this folder's one kustomization and registers it, which -// is the documented kustomize-root behaviour (and what the new-file-placement spec asserts -// directly). Before sibling inference was deleted, that resource was appended to the existing -// bundle instead, and the bundle was already listed — so the build file happened to stay -// byte-identical. Byte equality was therefore pinning a side effect of inference, not the -// property this spec is about: a hand-authored build file must not be reordered, reformatted, or -// have anything taken out of it. +// It replaces a byte-equality assertion, which pinned a side effect rather than the property this +// spec is about: a hand-authored build file must not be reordered, reformatted, or have anything +// taken out of it. A watched object with no document in Git is given a file beside the folder's +// kustomization and registered there, which is the documented kustomize-root behaviour. func expectKustomizationOnlyGainedResourceEntries(g Gomega, original, committed string) { GinkgoHelper() diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go index f94ec1bbb..36a35d93c 100644 --- a/test/e2e/quickstart_framework_e2e_test.go +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -138,9 +138,9 @@ func (r *quickstartFrameworkRun) helmInstallReadmeQuickstart() { chart = "charts/gitops-reverser" } - // ONE apply, exactly as the README's step 4 does it. This works because no webhook this chart - // installs is failurePolicy: Fail — the starter GitTarget is gated by nothing, so it is admitted - // while the Deployment it rolls is still coming up. Namespace authorization is enforced at + // ONE apply, exactly as the quickstart's install step does it. This works because the chart installs no + // admission webhook — the starter GitTarget is gated by nothing, so it is admitted while the + // Deployment it rolls is still coming up. Namespace authorization is enforced at // reconcile instead (checkSourceAuthorization, inside the Validated gate), which does not need // the manager to be reachable at admission time. args := []string{ @@ -177,7 +177,7 @@ func (r *quickstartFrameworkRun) helmInstallReadmeQuickstart() { // verifyNoRedisConfiguredAuthor asserts the controller logged the no-Redis // configured-author sentinel (main.go emits "configured-author mode: no Redis // configured" only when queue.redis.addr is empty) and that the pod is not -// crash-looping — proving admission-on-without-Redis is a healthy no-op. +// crash-looping — proving running without Redis is a healthy mode. func (r *quickstartFrameworkRun) verifyNoRedisConfiguredAuthor() { releaseNs := resolveE2ENamespace() release := resolveE2EInstallName(releaseNs) @@ -199,7 +199,7 @@ func (r *quickstartFrameworkRun) verifyNoRedisConfiguredAuthor() { for _, field := range strings.Fields(restarts) { count, convErr := strconv.Atoi(field) Expect(convErr).NotTo(HaveOccurred()) - Expect(count).To(Equal(0), "controller must not crash-loop with admission on and no Redis") + Expect(count).To(Equal(0), "controller must not crash-loop without Redis") } } diff --git a/test/e2e/source_cluster_e2e_test.go b/test/e2e/source_cluster_e2e_test.go index a1f6c5244..9888f6416 100644 --- a/test/e2e/source_cluster_e2e_test.go +++ b/test/e2e/source_cluster_e2e_test.go @@ -19,7 +19,7 @@ import ( // the provider's OWN namespace, or spec.inCluster where the installation allows it), for which // other namespaces may use it (spec.allowedGitTargets), and for its audit route. // -// Three kinds of spec live here: +// Four kinds of spec live here: // - Input-validation specs (Scenario 1) assert the KubeProvider reconciler's Validated verdict on // bad kubeconfigs and on an inCluster provider the installation does not list, without a // remote cluster and without a dial. @@ -31,6 +31,13 @@ import ( // Scenario 8 holds the SAME namespace + resource with different content in three workspaces, // proving state is keyed by source cluster, not (namespace, GVR); Scenario 9 gives two // providers ONE name in two namespaces, and recreates one of them. +// - The applier spec (Scenario 10) is an applier in this cluster applying into a remote one, as +// gitops-api runs: Flux applies a folder into a workspace through Kustomization.spec.kubeConfig, +// the GitTarget mirrors that workspace, and its FluxClaim holds the Kustomization through the +// install's own provider. It shows the startup converge landing in the workspace, an edit made +// there withheld from the apply's revert by the REMOTE object's managedFields, two +// CommitRequests that take over their edits' windows (the workspace sends no audit facts) and +// report the apply, and a CommitRequest created in the workspace itself. // // The whole suite is gated by skipUnlessSourceClusterEnabled() (env E2E_ENABLE_SOURCE_CLUSTER), // and the kcp specs additionally Skip when kcp is not installed — so a default `task test-e2e` @@ -635,4 +642,374 @@ spec: expectMirroredValue(g, repo.CheckoutDir, "clusters/twin-a", "twin", "alpha") }).WithTimeout(420 * time.Second).WithPolling(5 * time.Second).Should(Succeed()) }) + + // Scenario 10 — an applier in this cluster applying into the remote one: Flux applies a folder + // into a workspace through Kustomization.spec.kubeConfig, the GitTarget mirrors that workspace + // through a secretRef KubeProvider, and the FluxClaim holds the Kustomization through the + // install's own provider. So the apply window judges the managedFields of the REMOTE object. It + // has its own repository, so no GitTarget of the scenarios above shares its branch. + // + // The e2e kcp has no audit stream, so no write in the workspace names its author: everything is + // mirrored from the watch, and every event waits out the attribution grace first. + Describe("Flux applying into a workspace the GitTarget mirrors", Ordered, func() { + const ws = "sc-flux" + var ( + p *timingPipeline + r *echoRun + ksName string + // hash is the workspace's logical cluster. + hash string + ) + + BeforeAll(func() { + if kcp == nil { + Skip("kcp is not installed; run this corner via `task test-e2e-source-cluster`") + } + grantAppliersInAnyNamespace() + ensurePrometheusClient() + fluxRepo := SetupRepo(resolveE2EContext(), testNs, + fmt.Sprintf("e2e-source-cluster-flux-%d", GinkgoRandomSeed())) + p = newTimingPipeline(testNs, ws, fluxRepo) + ksName = ws + "-" + testNs + _, err := kubectlRunInNamespace(testNs, "apply", "-f", fluxRepo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply repo secrets") + configureRepoOriginWithCredentials(fluxRepo, testNs) + + By("provisioning the workspace, and the namespace the folder holds inside it") + DeferCleanup(kcp.deleteWorkspace, ws) + hash = kcp.createWorkspace(ws) + _, err = kcp.wsKubectl(hash, "create", "namespace", testNs) + Expect(err).NotTo(HaveOccurred(), "create the folder's namespace in %s", ws) + + By("mirroring the workspace: KubeProvider, GitProvider with a receiver, GitTarget, WatchRule") + DeferCleanup(func() { + deleteKubeProvider(testNs, ws) + cleanupNamespacedResource(testNs, "secret", ws+"-kubeconfig") + }) + writeKubeConfigSecret(testNs, ws+"-kubeconfig", "value", kcp.operatorKubeConfig(hash)) + _, err = applyKubeProvider(testNs, ws, ws+"-kubeconfig", "") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(p.cleanup) + createReadyGitProvider(p.providerName, testNs, fluxRepo.GitSecretHTTP, fluxRepo.RepoURLHTTP) + p.hookID = registerOperatorReceiver(testNs, p.receiverName, fluxRepo.RepoName) + _, err = applyGitTargetWithKubeProvider(testNs, p.targetName, p.providerName, "", p.gitPath, testNs, ws) + Expect(err).NotTo(HaveOccurred()) + verifyResourceCondition("gittarget", p.targetName, testNs, "SourceClusterReachable", "True", "", "", "180s") + p.applyWatchRule() + waitForStreamsRunning(p.targetName, testNs) + + r = &echoRun{ + timingPipeline: p, + applier: echoApplier{ + name: "Flux", + owns: func(g Gomega, ns, configMap string) bool { + out, err := kcp.wsKubectl(hash, "-n", ns, "get", "configmap", configMap, + `-o=jsonpath={.metadata.labels.kustomize\.toolkit\.fluxcd\.io/name}`) + g.Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(out) == ksName + }, + operation: func(g Gomega) claimOperation { + return readClaimOperation(g, testNs, fluxClaimResource, ksName) + }, + }, + kubectl: func(args ...string) (string, error) { + return kcp.wsKubectl(hash, append([]string{"-n", testNs}, args...)...) + }, + } + + By("creating a ConfigMap in the workspace, and waiting until the GitTarget mirrored it") + _, err = r.liveKubectl("create", "configmap", timingConfigMap, "--from-literal=value=seed") + Expect(err).NotTo(HaveOccurred()) + _, err = r.liveKubectl("label", "configmap", timingConfigMap, timingLabelKey+"=true") + Expect(err).NotTo(HaveOccurred()) + r.eventuallyEverywhere(timingConfigMap, "seed", echoApplyBudget) + + By("creating the Kustomization, which applies the folder into the workspace once") + DeferCleanup(func() { + cleanupNamespacedResource("flux-system", "kustomization", ksName) + cleanupNamespacedResource("flux-system", "gitrepository", ksName) + cleanupNamespacedResource("flux-system", "secret", ksName+"-git") + cleanupNamespacedResource("flux-system", "secret", ksName+"-kubeconfig") + }) + applyFluxIntoWorkspace(testNs, fluxRepo, ksName, kcp.operatorKubeConfig(hash), "./"+p.gitPath) + + By("pushing a Git-side change the Kustomization has not fetched, then holding it with a FluxClaim") + r.pushFromOutside("converged", "c1") + DeferCleanup(cleanupNamespacedResource, testNs, fluxClaimResource, ksName) + createFluxClaim(testNs, ksName, ksName) + nameApplierClaim(testNs, p.targetName, "FluxClaim", ksName, "") + }) + + // Naming the FluxClaim applies the tip, which holds a ConfigMap only Git has, since the + // GitRepository's 24h interval never fetched it. The reason is the ForeignCommit the receiver + // reported, which the worker kept while no applier was named, or the startup CatchUp. So the + // ConfigMap reaches the workspace through the FluxClaim's apply alone, and Flux's labels on it + // say who applied it. + It("applies its startup converge into the workspace, and mirrors what it applied", func() { + verifyResourceCondition(fluxClaimResource, ksName, testNs, "Configured", "True", "", "") + verifyResourceCondition(fluxClaimResource, ksName, testNs, "Ready", "True", "Succeeded", "") + verifyResourceCondition("gittarget", p.targetName, testNs, "ApplierClaimReady", "True", "Succeeded", "") + Eventually(func(g Gomega) { + head := p.remoteBranchHead(g) + g.Expect(fluxLastAppliedRevision(g, ksName)).To(ContainSubstring(head)) + done := r.applier.operation(g) + g.Expect(done.Reason).To(BeElementOf("ForeignCommit", "CatchUp")) + g.Expect(done.Commit).To(Equal(head)) + g.Expect(done.Phase).To(Equal("Succeeded")) + live, exists := r.liveValue(g, "converged") + g.Expect(exists).To(BeTrue(), "the converge applies the Git-side change into the workspace") + g.Expect(live).To(Equal("c1")) + g.Expect(r.applier.owns(g, testNs, "converged")).To(BeTrue(), "Flux applied it") + g.Expect(r.applier.owns(g, testNs, timingConfigMap)).To(BeTrue(), "Flux adopted the mirrored one") + }, echoApplyBudget, 2*time.Second).Should(Succeed()) + verifyResourceCondition(fluxClaimResource, ksName, testNs, "InControl", "True", "Suspended", "") + r.consistentlyEverywhere("converged", "c1") + r.consistentlyEverywhere(timingConfigMap, "seed") + }) + + // The collision the apply window exists for: an edit the apply reverts. E5 edits once the apply + // was asked for, and races Flux, which often writes first. Here every event from the workspace + // waits out the attribution grace before it reaches the worker (kcp sends no audit facts), and + // that makes the collision certain: an edit made just before a Git-side push reaches the worker + // only after the push was heard, its apply asked for, and Flux applied it. The commit Flux + // applied does not hold the edit, so Flux reverts it in the workspace. Its write reaches the + // worker after the edit, later still, and the window, which waits until the watch delivered every + // write made before the apply completed, withholds it: the object's managedFields name + // kustomize-controller alone, after the edit. Git keeps the edit, and a Repair puts it back. + // + // It holds while the attribution grace (10s in e2e) outlasts the time from the push to the apply + // request, which the receiver makes a second or two. + It("withholds an apply that reverts an edit made in the workspace", func() { + withheld := fmt.Sprintf(`sum(gitopsreverser_apply_window_collisions_total{gitbranch_namespace=%q,`+ + `gitbranch_name=%q,outcome="withheld"}) or vector(0)`, testNs, gitBranchName(p.providerName, "main")) + before, err := queryPrometheus(withheld) + Expect(err).NotTo(HaveOccurred()) + + r.pushFromOutside("during", "d0") + r.eventuallyEverywhere("during", "d0", echoApplyBudget) + + By("editing an object in the workspace, and pushing a Git-side change to another before its event arrives") + token := r.applier.operation(Default).Token + r.editThroughAPI("during", "d-edit") + r.pushFromOutside("during-other", "o1") + Eventually(func(g Gomega) { + g.Expect(r.applier.operation(g).Token).NotTo(Equal(token), "the apply is asked for") + }, echoApplyBudget, 250*time.Millisecond).Should(Succeed()) + + r.eventuallyEverywhere("during", "d-edit", echoApplyBudget) + r.eventuallyEverywhere("during-other", "o1", echoApplyBudget) + r.consistentlyEverywhere("during", "d-edit") + waitForMetricWithTimeout(withheld, func(v float64) bool { return v > before }, + "the apply over the workspace edit is counted as a withheld collision", time.Minute) + }) + + // A save made in the workspace itself, next to the edit: a CommitRequest created there, served + // once the workspace's KubeProvider reconciles them and the CRD is installed. Before that, a + // request on this cluster for the GitTarget is refused, since the GitTarget mirrors the workspace. + // The GitTarget must admit the request by the workspace namespace: one is refused while it admits + // none. kcp sends no audit facts, so the request names nobody, like the edit, and attaches to the + // edit's window under the ordinary rule. Its own author is looked for first, for the whole + // attribution grace, so the edit is made once the request is waiting for a window. + It("saves an edit made in the workspace with a CommitRequest created there", func() { + By("refusing a CommitRequest on this cluster for a GitTarget that mirrors the workspace") + const here = "sc-flux-save-here" + applyCommitRequest(testNs, here, p.targetName, "e2e: made on the wrong cluster") + Eventually(func(g Gomega) { + g.Expect(commitRequestCondition(g, testNs, here, "Ready")).To(Equal("False")) + g.Expect(commitRequestField(g, testNs, here, `{.status.conditions[?(@.type=="Ready")].reason}`)). + To(Equal("NotAllowed")) + g.Expect(commitRequestField(g, testNs, here, `{.status.conditions[?(@.type=="Ready")].message}`)). + To(ContainSubstring("create the CommitRequest there")) + }, 2*time.Minute, time.Second).Should(Succeed()) + + By("installing the CommitRequest CRD in the workspace, and reconciling the workspace's CommitRequests") + _, err := kcp.wsKubectl(hash, "apply", "--server-side", "-f", + "config/crd/bases/configbutler.ai_commitrequests.yaml") + Expect(err).NotTo(HaveOccurred()) + _, err = kcp.wsKubectl(hash, "wait", "--for=condition=Established", + "crd/commitrequests.configbutler.ai", "--timeout=60s") + Expect(err).NotTo(HaveOccurred()) + _, err = kubectlRunInNamespace(testNs, "patch", "kubeprovider", ws, "--type=merge", "-p", + `{"spec":{"commitRequests":"ReconcileAndDeleteFinished"}}`) + Expect(err).NotTo(HaveOccurred()) + verifyResourceCondition("kubeprovider", ws, testNs, "CommitRequestsServed", "True", "Watching", "", "120s") + + By("refusing a CommitRequest while the GitTarget admits none from the workspace") + const refused = "sc-flux-workspace-save-refused" + createWorkspaceCommitRequest(kcp, hash, testNs, refused, "e2e: not admitted", testNs, p.targetName, "30s") + Eventually(func(g Gomega) { + g.Expect(workspaceCommitRequestField(g, kcp, hash, testNs, refused, + `{.status.conditions[?(@.type=="Ready")].status}`)).To(Equal("False")) + g.Expect(workspaceCommitRequestField(g, kcp, hash, testNs, refused, + `{.status.conditions[?(@.type=="Ready")].reason}`)).To(Equal("NotAllowed")) + }, 2*time.Minute, time.Second).Should(Succeed()) + + By("admitting the workspace namespace's CommitRequests on the GitTarget") + _, err = kubectlRunInNamespace(testNs, "patch", "gittarget", p.targetName, "--type=merge", "-p", + fmt.Sprintf(`{"spec":{"allowedCommitRequests":[{"sourceNamespaces":[%q]}]}}`, testNs)) + Expect(err).NotTo(HaveOccurred()) + + By("creating a CommitRequest in the workspace, and editing once it waits for a window") + const name, message = "sc-flux-workspace-save", "e2e: save from the workspace" + createWorkspaceCommitRequest(kcp, hash, testNs, name, message, testNs, p.targetName, "30s") + saveField := func(g Gomega, jsonPath string) string { + return workspaceCommitRequestField(g, kcp, hash, testNs, name, jsonPath) + } + Eventually(func(g Gomega) { + g.Expect(saveField(g, `{.status.conditions[?(@.type=="Ready")].status}`)).To(Equal("Unknown")) + }, 2*time.Minute, time.Second).Should(Succeed(), "the request is claimed and waits for a window") + r.editThroughAPI(timingConfigMap, "saved-1") + + By("the save commits the edit, and reports the commit, the apply and its deletion time on the request") + var commit string + Eventually(func(g Gomega) { + g.Expect(saveField(g, `{.status.conditions[?(@.type=="Ready")].status}`)).To(Equal("True")) + g.Expect(saveField(g, `{.status.conditions[?(@.type=="Ready")].reason}`)).To(Equal("Committed")) + commit = saveField(g, "{.status.commit}") + g.Expect(commit).NotTo(BeEmpty()) + g.Expect(saveField(g, "{.status.author}")).To(BeEmpty(), "kcp sends no audit facts") + g.Expect(saveField(g, `{.metadata.annotations.configbutler\.ai/delete-after}`)).NotTo(BeEmpty(), + "ReconcileAndDeleteFinished stamps a finished request for deletion") + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + committed, inGit := r.committedValue(Default, timingConfigMap) + Expect(inGit).To(BeTrue()) + Expect(committed).To(Equal("saved-1")) + body, err := gitRun(p.repo.CheckoutDir, "show", "-s", "--format=%B", commit) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(body)).To(Equal(message)) + + Eventually(func(g Gomega) { + g.Expect(saveField(g, `{.status.conditions[?(@.type=="Applied")].status}`)).To(Equal("True")) + g.Expect(saveField(g, `{.status.conditions[?(@.type=="Applied")].reason}`)).To(Equal("Applied")) + g.Expect(saveField(g, "{.status.appliedRevision}")).To(ContainSubstring(commit)) + g.Expect(fluxLastAppliedRevision(g, ksName)).To(ContainSubstring(commit)) + }, echoApplyBudget, 2*time.Second).Should(Succeed()) + r.consistentlyEverywhere(timingConfigMap, "saved-1") + }) + + // Saves on the workspace's GitTarget that ask to catch up are applied once they are pushed, and + // each request reports the apply: Applied=True, with the revision Flux applied. The second save + // is made as soon as the first is pushed, so the two may share one apply. Every event from the + // workspace waits out the attribution grace first, so the requests wait longer than that for their + // window. + It("saves edits in the workspace, and reports the apply on each of two saves", func() { + names := []string{"sc-flux-save-1", "sc-flux-save-2"} + field := func(g Gomega, name, jsonPath string) string { + return workspaceCommitRequestField(g, kcp, hash, testNs, name, jsonPath) + } + commits := map[string]string{} + for i, name := range names { + value := fmt.Sprintf("saved-%d", i+2) + createWorkspaceCommitRequest(kcp, hash, testNs, name, "e2e: save "+name+" and apply it", testNs, + p.targetName, "45s") + r.editThroughAPI(timingConfigMap, value) + Eventually(func(g Gomega) { + g.Expect(field(g, name, `{.status.conditions[?(@.type=="Ready")].status}`)).To(Equal("True")) + g.Expect(field(g, name, `{.status.conditions[?(@.type=="Ready")].reason}`)). + To(Equal(commitRequestReasonCommitted)) + commits[name] = field(g, name, "{.status.commit}") + g.Expect(commits[name]).NotTo(BeEmpty()) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + committed, inGit := r.committedValue(Default, timingConfigMap) + Expect(inGit).To(BeTrue()) + Expect(committed).To(Equal(value), "the save's commit holds its edit") + } + + last := commits[names[len(names)-1]] + Eventually(func(g Gomega) { + for _, name := range names { + applied := field(g, name, `{.status.conditions[?(@.type=="Applied")].status}`) + reason := field(g, name, `{.status.conditions[?(@.type=="Applied")].reason}`) + if applied == "False" { + StopTrying(fmt.Sprintf("CommitRequest %s is Applied=False (%s): only a commit the branch "+ + "lost may be", name, reason)).Now() + } + g.Expect(applied).To(Equal("True"), "CommitRequest %s Applied (%s)", name, reason) + g.Expect(reason).To(Equal("Applied")) + // A later save's commit holds the earlier one's, so one apply may settle both. + g.Expect(field(g, name, "{.status.appliedRevision}")). + To(Or(ContainSubstring(commits[name]), ContainSubstring(last))) + } + g.Expect(field(g, names[len(names)-1], "{.status.appliedRevision}")).To(ContainSubstring(last)) + g.Expect(fluxLastAppliedRevision(g, ksName)).To(ContainSubstring(last)) + }, echoApplyBudget, 2*time.Second).Should(Succeed()) + r.eventuallyEverywhere(timingConfigMap, "saved-3", echoApplyBudget) + }) + }) }) + +// createWorkspaceCommitRequest creates the CommitRequest name in namespace of a workspace, saving the +// GitTarget targetNS/target and asking to catch up, as JSON. It waits up to attachTimeout for a window: +// an edit from a cluster with no audit stream reaches its window only after the attribution grace. +func createWorkspaceCommitRequest( + kcp *kcpTunnel, hash, namespace, name, message, targetNS, target, attachTimeout string, +) { + GinkgoHelper() + f, err := os.CreateTemp("", "e2e-commitrequest-*.json") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.Remove(f.Name()) }() + _, err = fmt.Fprintf(f, `{"apiVersion": "configbutler.ai/v1alpha3", "kind": "CommitRequest", +"metadata": {"name": %q, "namespace": %q}, +"spec": {"gitTargetRef": {"namespace": %q, "name": %q}, "message": %q, "catchUp": "AfterPush", +"window": {"attachTimeout": %q, "maxDuration": "4s"}}} +`, name, namespace, targetNS, target, message, attachTimeout) + Expect(err).NotTo(HaveOccurred()) + Expect(f.Close()).To(Succeed()) + _, err = kcp.wsKubectl(hash, "create", "-f", f.Name()) + Expect(err).NotTo(HaveOccurred(), "failed to create CommitRequest %s/%s in the workspace", namespace, name) +} + +// workspaceCommitRequestField reads a jsonpath field off a CommitRequest in a workspace. +func workspaceCommitRequestField(g Gomega, kcp *kcpTunnel, hash, namespace, name, jsonPath string) string { + out, err := kcp.wsKubectl(hash, "-n", namespace, "get", "commitrequest", name, "-o", "jsonpath="+jsonPath) + g.Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(out) +} + +// applyFluxIntoWorkspace creates, in flux-system, a GitRepository following main and a Kustomization +// applying path from it into a kcp workspace: its spec.kubeConfig names a Secret holding kubeconfig +// under the key "value", which kustomize-controller, in this cluster, dials. Both are on a 24h +// interval, so a later commit is applied only when the FluxClaim asks. It waits for the first apply, +// made before any applier claim holds the Kustomization. +func applyFluxIntoWorkspace(testNs string, repo *RepoArtifacts, name, kubeconfig, path string) { + GinkgoHelper() + writeKubeConfigSecret("flux-system", name+"-kubeconfig", "value", kubeconfig) + fields := struct { + Namespace, SecretName, Name, RepoURL, Branch, Interval, Username, Password string + }{ + Namespace: "flux-system", SecretName: name + "-git", Name: name, RepoURL: repo.RepoURLHTTP, + Branch: "main", Interval: "24h", + } + checkout := gitCheckout{secretNamespace: testNs, gitSecretName: repo.GitSecretHTTP} + fields.Username, fields.Password = checkout.readGitCredentialSecretDataBase64() + Expect(applyFromTemplate("test/e2e/templates/bi-directional/flux-gitrepository-http.tmpl", fields, + "flux-system")).To(Succeed(), "failed to apply the Flux GitRepository") + _, err := kubectlRunWithStdin("flux-system", fmt.Sprintf(`apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: %s + namespace: flux-system +spec: + interval: 24h + timeout: 2m + path: %s + prune: true + sourceRef: + kind: GitRepository + name: %s + kubeConfig: + secretRef: + name: %s-kubeconfig + key: value +`, name, path, name, name), "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to apply the Flux Kustomization") + // Ready's message is where an unreachable workspace shows: a TLS or DNS failure to the front-proxy. + Eventually(func(g Gomega) { + out, err := kubectlRunInNamespace("flux-system", "get", "kustomization", name, "-o", + `jsonpath={.status.conditions[?(@.type=="Ready")].status}|`+ + `{.status.conditions[?(@.type=="Ready")].message}`) + g.Expect(err).NotTo(HaveOccurred()) + status, message, _ := strings.Cut(strings.TrimSpace(out), "|") + g.Expect(status).To(Equal("True"), "Kustomization %s is not Ready: %s", name, message) + }, 3*time.Minute, 2*time.Second).Should(Succeed(), "the Kustomization must apply its folder into the workspace") +}