diff --git a/.docs-lint-scope b/.docs-lint-scope index ef4b57fd0..770d40cbe 100644 --- a/.docs-lint-scope +++ b/.docs-lint-scope @@ -24,6 +24,7 @@ README.md docs/architecture.md +docs/interpreting-metrics.md docs/configuration.md docs/bi-directional-flux.md docs/bi-directional-argocd.md diff --git a/cmd/main.go b/cmd/main.go index 0c964e41f..0395a8f18 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -99,7 +99,7 @@ func main() { // Initialize metrics setupCtx := ctrl.SetupSignalHandler() - _, err := telemetry.InitOTLPExporter(setupCtx) + _, err := telemetry.InitPrometheusExporter(setupCtx) fatalIfErr(err, "unable to initialize metrics exporter") // TLS/options diff --git a/docs/TODO.md b/docs/TODO.md index 9c817f43b..4519174bd 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -20,7 +20,7 @@ Nothing open. The 0.54.0 release PR's changelog was checked against what ships o discard or race an open commit window or pending live edits ([render-fidelity.md](design/support-boundary/render-fidelity.md) §6b). - [ ] **Watch the held applier**, so a switch-on or a `flux resume` is undone at once instead of - within the one-minute `heldClaimSteadyInterval`. Also makes `claimed_resource_reconciled_total` + within the one-minute `heldClaimSteadyInterval`. Also makes `apply_claimed_resource_reconciled_total` (ask #31) count at the moment of the resume. [applier-claim-held-applier-watch.md](design/applier-claim-held-applier-watch.md). - [ ] **F3: a Flux fetch is made before the hold is checked**, so a `Kustomization` resumed by hand diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 28d8a890a..176c54eaa 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -187,8 +187,53 @@ A `WatchRule` or `ClusterWatchRule` no longer reads the GitProvider and no longe Deleting a `GitBranch` stops its branch worker after it has pushed or refused every pending write and settled every `CommitRequest` it held. It never deletes the branch on the remote. -The metrics keep their `provider_namespace` and `provider_name` labels, which name the `GitProvider` -of the `GitBranch` holding the branch. Log lines name a branch as `repository@branch`. +The branch worker's metrics are labeled by the `GitBranch`; see the metrics entry below. Log lines +name a branch as `repository@branch`. + +## Metrics: the branch worker's series name their `GitBranch`, and three apply series are renamed + +**Every series a branch worker records carries `gitbranch_namespace` and `gitbranch_name`, and no +longer carries `branch`.** `provider_namespace` and `provider_name` stay. That is the +`git_commits_total`, `git_pushes_total`, `git_push_retries_total`, `git_push_duration_seconds`, +`git_fetches_total`, `git_queue_refusals_total`, `git_commit_failures_total`, +`git_writes_dropped_total` and `git_materialization_failures_total` counters; the +`git_queue_depth`, `git_retained_bytes`, `git_retained_writes`, `git_intake_paused`, +`git_oldest_retained_write_timestamp_seconds` and `git_next_retry_timestamp_seconds` gauges; and +the apply window and echo series below. Select a branch by its `GitBranch`: + +```promql +# 0.53.0: gitopsreverser_git_pushes_total{provider_namespace="platform", branch="main"} +gitopsreverser_git_pushes_total{gitbranch_namespace="platform", gitbranch_name="platform-main"} +``` + +`git_branch_targets` maps each `GitTarget` to its `GitBranch` (`gitbranch_namespace`, +`gitbranch_name`) and no longer carries the provider or the branch. + +**Three apply series are renamed** so that every apply series starts with `apply_`: + +| 0.53.0 | Now | +| --- | --- | +| `appliers_held` | `apply_appliers_held` | +| `stale_snapshots_total` | `apply_stale_snapshots_total` | +| `claimed_resource_reconciled_total` | `apply_claimed_resource_reconciled_total` | + +**Labels that changed elsewhere:** + +- `git_resync_failures_total` gains `reason`: `error` and `timeout` are a snapshot that did not + land; `refused` and `deferred` are counted too, and lose nothing. Alert on + `reason=~"error|timeout"`. +- `git_fetches_total` no longer has `reason="bootstrap"`, which never emitted. +- `git_commit_failures_total` and `git_writes_dropped_total` no longer carry `kind`, and + `git_commit_windows_total` no longer has `close_reason="atomic_before_apply"`: the atomic write + path they described had no producer, and is gone. +- `attribution_resolution_wait_seconds` carries only `tier` and `event_kind`; the type is on + `attribution_resolutions_total`. +- `apply_requests_total` and `apply_revision_mismatches_total` gain `gitbranch_namespace`, + `gitbranch_name`, `provider_namespace`, `provider_name`, `claim_namespace` and `claim_name`; + `audit_events_total` gains `audit_route`; `secret_encryptions_total` gains + `gittarget_namespace` and `gittarget_name`; `watch_sessions_ended_total`, + `watch_replay_duration_seconds` and `watch_event_handling_seconds` gain `source_cluster`. A query + that sums over these reads as before. ## Grants say who may use a provider or a branch; `allowedBranches` is `allowedGitBranches` diff --git a/docs/api-first-publication.md b/docs/api-first-publication.md index 7775d79d8..2ef82ddf7 100644 --- a/docs/api-first-publication.md +++ b/docs/api-first-publication.md @@ -467,7 +467,6 @@ provider and branch so different workers do not hide one another's behavior. | Same counter, reason `push_failure_probe` | Remote-state lookup after an unclassified push failure | | Same counter, reason `recovery` | Attempts to rebuild unsafe retained work or clean a dirty checkout | | Same counter, reason `forced_recheck` | Explicit refresh, and the snapshot a resync judges against | -| Same counter, reason `bootstrap` | Instrumented bootstrap helper; currently called only by tests | | `gitopsreverser_git_pushes_total` and `gitopsreverser_git_push_retries_total` | Completed publication cycles and contention retries within them | | `gitopsreverser_git_push_duration_seconds` | Time inside a push cycle, including its inline replays | | `gitopsreverser_git_queue_depth` and `gitopsreverser_git_queue_refusals_total` | Outstanding work and queue overload | diff --git a/docs/architecture.md b/docs/architecture.md index 3d65fc1e0..618993b1b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,8 +21,9 @@ write boundary: its credential, branch policy, and targets usually belong togeth `KubeProvider` is one **logical source identity**: one client, discovery surface, watch state, and attribution partition, shared by every target that names it. Its kubeconfig Secret lives in its own namespace, so a tenant's connection keeps its credential with the tenant. Its grants -(`spec.allowedGitTargets`, `spec.allowedApplierClaims`) admit control-cluster namespaces, its own -included only by a grant, and bound what they may reach; none grants source-cluster RBAC. The source identity is the provider's +(`spec.allowedGitTargets`, `spec.allowedApplierClaims`) admit control-cluster namespaces and bound +what they may reach; omitted, each admits the provider's own namespace, and none grants +source-cluster RBAC. The source identity is the provider's namespace and name, with no API-server identity probe: two providers configured for the same server remain separate source partitions, and so do two providers with one name in two namespaces. @@ -81,7 +82,7 @@ fails; there is no plaintext opt-out. **Writes are serialized per Git branch.** One [BranchWorker](../internal/git/branch_worker.go) owns each repository branch: the repository's URL, normalized, and the branch. Multiple `GitTarget`s may share -one branch, from any namespace, through the one `GitProvider` that owns it. Every write to that +one branch, from any namespace, through the one `GitBranch` that holds it. Every write to that branch goes through the worker's single event loop and commit window. **The worktree sits at the remote tip, or the worker knows it does not.** The write path used to @@ -96,7 +97,7 @@ which with writes retained says their local commits are gone and must be replaye two would let one kind of doubt clear another. Either one makes the next cycle fetch. The failure direction is safe by construction: a stale "untrusted" costs one fetch, while a stale "trusted" is caught by the compare-and-swap on the next push. For a new write branch, the worker checks the -parent's advertised tip before sending the push. A moved parent applier claims fetch, reset, and replay. +parent's advertised tip before sending the push. A moved parent costs a fetch, a reset, and a replay. The server's compare-and-swap protects creation of the write branch with `Old = zero`; it does not lock the parent. The parent can still move after the advertisement, so what is guaranteed is the parent's tip as that advertisement showed it. **The corollary is the rule to keep in your head @@ -395,8 +396,9 @@ event serves only to name who scaled the parent. One `KubeProvider` grant answers two questions in different planes: -- `from` (with `names` or `selector`) authorizes **control-cluster** namespaces to reference the - provider. It is a tenant/export boundary; it neither selects nor grants access to source namespaces. +- `consumerNamespaces` (`from`, with `names` or `selector`) authorizes **control-cluster** + namespaces to reference the provider. It is a tenant/export boundary; it neither selects nor + grants access to source namespaces. - `sourceNamespaces` and `clusterResources` bound what the `GitTarget`s it admits may watch. They grant nothing: what may actually be read from the source cluster is bounded by that credential's own Kubernetes RBAC. The operator reads no `Namespace` objects in a source cluster for @@ -464,9 +466,7 @@ exactly one target. `gitBranchRef`, `kubeProviderRef`, and `path` are immutable so a target cannot silently orphan an old materialization or change its source cluster. Among GitTargets on one `GitBranch`, from any -namespace, the controller rejects overlapping paths. `spec.gitProviderRef`, `spec.branch` and -`spec.parentBranch` left the `GitTarget` for the `GitBranch`; the schema keeps them only so that -setting one is refused, naming `gitBranchRef`. +namespace, the controller rejects overlapping paths. A `GitTarget` whose `GitBranch` does not exist reports `Validated=False` (`GitBranchNotFound`); one whose `GitBranch` does not hold its branch, or is not `Ready`, reports `GitBranchReady=False` and @@ -485,6 +485,9 @@ Status has a kstatus-compatible summary layer plus domain conditions: source API the data plane can currently reach. - `GitPathAccepted` explains the target side: the selected Git path is safe for the operator to materialize. +- `GitBranchReady` says the `GitBranch` holds its branch and is `Ready`. +- `RenderMatchesLive` says the folder, as rendered, agrees with the live objects; while it is not + `True`, live writes are held or dropped (see [The render-vs-live gate](#the-render-vs-live-gate)). - `ApplierClaimReady`, present only with `spec.applierClaimRef`, says the applier claim is `Ready` (its applier exists, can be written, and is held: a `Kustomization` suspended, an `Application`'s automated sync off) and that the @@ -539,8 +542,8 @@ it. Every field but `spec.allowedGitTargets` is immutable: it does not exist. Omitted means the remote's default branch. An explicit parent must exist when creating the write branch (`ParentBranchNotFound`). The grants need not allow it, because it is only read. -- `spec.allowedGitTargets`: which namespaces' `GitTarget`s may write through it, its own included - only by a grant. +- `spec.allowedGitTargets`: which namespaces' `GitTarget`s may write through it. Omitted, its own + namespace's; a written list replaces that default, and `[]` admits none. - `status.repository`: the normalized repository URL, half of the key one `GitBranch` holds. - `status.lastPushNoticeAt`: when a `GitReceiver` last delivered a push to it. It records receipt, so it is published even when the look the delivery caused could not read the remote. @@ -603,7 +606,8 @@ Several namespaces may mirror through one provider, but its source identity must because it keys source clients, discovery, watches, and attribution. The identity is the provider's `namespace/name` rather than a deduplicated physical-cluster identity, so two providers pointing at the same API server still have separate contexts and authorization boundaries. A provider admits only -the namespaces its grants admit, its own included only by a grant. The check runs on every reconcile +the namespaces its grants admit; omitted, the grants admit its own namespace, and a written list +replaces that default. The check runs on every reconcile before watches start (`NamespaceNotAuthorized`), and a grant or Namespace label change triggers that reconcile, so tightening it also stops an already-existing `GitTarget`, which an admission-time check could not do. It guards which tenant may cause the operator to export a source; it does not expand that source @@ -613,8 +617,12 @@ Its audit route, `spec.attribution.auditRoute`, defaults to `.` namespace: the namespace the route's first dot-segment names when a provider there carries it, otherwise the namespace of the oldest carrier. Providers in the holding namespace may share a route (several connections to one API server); one in another namespace reports `Stalled=True` -(`AuditRouteConflict`) and is not used. The `KubeProvider` validates its connection inputs, while a -`GitTarget` projects provider readiness and live source reachability. +(`AuditRouteConflict`) and is not used. `AuditFactsReceived` reports whether the route has ever +delivered a fact: `Unknown` until one arrives, and it is never part of `Ready`, because a quiet +cluster and a broken audit policy look the same from here +([configuration.md](configuration.md#audit-route-and-auditfactsreceived)). The `KubeProvider` +validates its connection inputs, while a `GitTarget` projects provider readiness and live source +reachability. Credentials are re-resolved on the catalog-refresh cadence, not on every watch reconnect, `inCluster` ones included. A rotated Secret, a `spec.client` change, or a provider deleted and recreated under the @@ -731,7 +739,7 @@ flowchart TD RESOLVED --> WINDOW UNRESOLVED --> WINDOW - WINDOW --> TIMER{Close applier claim} + WINDOW --> TIMER{What closes it?} TIMER -->|silence timer| BASE TIMER -->|buffer limit or resync boundary| BASE @@ -847,7 +855,11 @@ still the tip), writes the operation through to the applier claim's `status.operation`, and only then resumes the `Kustomization` the applier claim holds suspended, with a `requestedAt` token. It follows the apply until that token is handled, suspends the `Kustomization` again with a second token, and reports the operation complete once that one is handled -([`apply-trigger.md`](design/apply-trigger.md#the-flux-applier-suspended)). It never retries: +([`apply-trigger.md`](design/apply-trigger.md#the-flux-applier-suspended)). For an Argo CD +`Application`, whose automated sync the claim holds off, the dispatcher checks that its single +`spec.source` is this repository and branch, then sets the `Application`'s operation to a sync +pinned to that commit, never while another operation is set; the operation is complete when Argo CD +clears it ([the Argo CD applier](design/apply-trigger.md#the-argo-cd-applier)). It never retries: an attempt that does not deliver gives the worker back what it carried, a refusal only while it still stands, and the worker decides the retry after a flush, once that applier's backoff has run out. A delivered request records an `ApplyRequested` Event, and the worker records the operation on the applier claim's @@ -1059,16 +1071,15 @@ gap would otherwise win. There is no Redis call on this path: the fast case is a The resolver takes the strongest evidence available, and the tier it used is the `tier` label on `attribution_resolutions_total`. Who that evidence named is the separate `actor_kind` label, in the -same `user`/`serviceaccount`/`none` vocabulary `commits_total{author_kind}` uses: +same `user`/`serviceaccount`/`none` vocabulary `git_commits_total{author_kind}` uses: | Tier | Key | What it asserts | |---|---|---| | `delete_sticky` | uid (a delete fact, sticky) | who asked for this object's deletion | | `exact` | uid + rv | this actor produced this exact version | | `deletecollection_body_uid` | uid in a collection fact's set | the API server said this request deleted this object | -| `latest` | uid (the object's own delete fact) | who removed it | +| `latest` | uid (the object's own delete fact) | who removed it, when the sticky slot was evicted | | `name` | namespace + name | the same, for a fact with no uid (an aggregated API's usual shape) | -| `latest` | uid (a delete fact) | the object's own delete, when the sticky slot was evicted | | `deletecollection_scope` | namespace + selector + window | a collection request covering it was made | | `resource_version` | rv (the escape hatch for a fact with no uid) | a fact for this exact version, unidentified | | `absent` | none | nothing usable arrived in time | @@ -1280,9 +1291,10 @@ like an empty API surface, and a group/version that vanishes from a complete sca rather than being pruned. Both protect against accidental Git deletions on a discovery blink (see [typeset-owns-discovery-grace.md](spec/typeset-owns-discovery-grace.md)). Every active source context has its own catalog and registry, so a CRD served only by `prod-eu-1` is never resolved for a target mirroring -`prod-us-1`. Catalogs refresh at startup and on the 30-second reconciliation cadence. CRD/APIService -applier claim informers run only in the **control plane**, where the operator's own CRDs live; a remote source's -API-surface change is discovered on that periodic refresh. A target's plan pass never calls discovery +`prod-us-1`. Catalogs refresh at startup and on the 30-second reconciliation cadence. The +`CustomResourceDefinition` and `APIService` informers run only in the **control plane**, where the +operator's own CRDs live; a remote source's API-surface change is discovered on that periodic +refresh. A target's plan pass never calls discovery itself; for a source nothing has observed yet, it asks for a refresh at once. ### TypeRegistry and followability @@ -1331,8 +1343,8 @@ The watch manager is a controller runtime `Runnable` (`NeedLeaderElection`). It discovery, the per-GitTarget watch sets, and the watched type tables for GitTargets. Its object-state intake is the watches themselves. Every active `KubeProvider` has an independent source context; its catalog, registry, dynamic client, and reachability can fail or recover without borrowing facts from another -source. Discovery applier claim informers (CRDs / APIServices) run only in the control plane and accelerate its -catalog refresh; remote source catalogs use the periodic refresh. +source. Discovery informers (on `CustomResourceDefinition` and `APIService`) run only in the control +plane and accelerate its catalog refresh; remote source catalogs use the periodic refresh. On `Start` it bootstraps the RuleStore, refreshes the API catalog and TypeRegistry, resolves rule selection, and builds the collection watch plan. @@ -1366,11 +1378,11 @@ next plan pass resolves its claimed `(GVR, scope)` set against the source's curr a new rule starts watching gets a new watch opened (with a `sendInitialEvents` replay) and a type no longer claimed has its watch closed. -A **control-plane** CRD / APIService change reaches the same re-resolution through the watch manager -instead. Its applier claim informers wake a catalog refresh, which re-resolves the watched type tables and -brings the running watches into line. The GitTarget controller does not watch those objects. A remote -CRD / APIService change has no applier claim informer at all; the next periodic source-catalog refresh -performs the same re-resolution. +A **control-plane** `CustomResourceDefinition` or `APIService` change reaches the same +re-resolution through the watch manager instead. Its informers wake a catalog refresh, which +re-resolves the watched type tables and brings the running watches into line. The GitTarget +controller does not watch those objects. A remote `CustomResourceDefinition` or `APIService` change +has no informer at all; the next periodic source-catalog refresh performs the same re-resolution. ### Mark and sweep resync @@ -1424,6 +1436,39 @@ fresh one is asked for once Git is no longer ahead; it would otherwise revert th used anyway. See [stale snapshots](spec/reconcile-via-watchlist-mark-and-sweep.md#stale-snapshots-are-discarded). +### The render-vs-live gate + +- **Gate state, per collection**: [internal/watch/render_fidelity_gate.go](../internal/watch/render_fidelity_gate.go) +- **Writer side and the hold**: [internal/git/render_fidelity_gate.go](../internal/git/render_fidelity_gate.go), + [internal/git/render_fidelity_hold.go](../internal/git/render_fidelity_hold.go) +- **Design**: [render-fidelity.md](design/support-boundary/render-fidelity.md) + +The operator renders a folder with its own kustomize build, but Flux or Argo CD may apply it with +context the folder does not hold: a Flux `postBuild.substitute` value for a `${var}` token, or an +Argo CD `spec.source.kustomize` override. Mirroring a live value over such a token would replace +the token in Git with whatever the applier substituted. So every `GitTarget` carries a gate: +each collection's replay compares the rendered `${...}` values of the documents it governs with +the sanitized live objects, and the gate opens only once every collection of the target has +reported a clean replay under its current revision. The gate is per `GitTarget`, so one collection +that is still replaying closes live writes for all of them. `RenderMatchesLive` publishes it: + +- **`Unknown` (`Rechecking`): a collection is new, restarted, or still replaying**, which is every + collection at operator start. A live event that arrives is judged at once and then **held**; an + atomic write is dropped at once. + When the gate opens, the held events are released in arrival order, before any later event, and + committed with their authors. An applied snapshot of a collection supersedes the events of that + collection held before it. The hold is bounded, at 10 minutes from the first held event and by + the branch's retained-byte budget. Past either, the held events are dropped and a fresh snapshot + is asked for, which commits the current state as the operator. A delete among the dropped events + comes back only under `prune.mode: Always`. +- **`False` (`RenderDoesNotMatchLive`): a rendered token disagrees with live.** Live and atomic + writes are dropped and the target is `Stalled`. Scoped resyncs still run, so a fresh replay can + measure the folder again. A repair in Git does not reopen the gate on its own; a replay of the + target's collections that measures them clean does. + +Every dropped write is counted in `git_writes_dropped_total{reason}` (`render_fidelity_pending` for +a hold that gave up, `render_fidelity_diverged` for a divergence). Nothing replays a dropped write. + *** ## Git write architecture @@ -1753,21 +1798,27 @@ by the validating webhook, not derived from the audit attribution index. Controllers watch their dependencies so dependents reconcile quickly after spec changes: -- `GitTargetReconciler` watches `GitProvider`, `KubeProvider`, `Namespace`, `WatchRule`, and - `ClusterWatchRule`. Provider changes re-check connection and authorization inputs; control-cluster +- `GitTargetReconciler` watches `GitBranch`, `GitProvider`, `KubeProvider`, `Namespace`, `WatchRule`, + and `ClusterWatchRule`. Provider changes re-check connection and authorization inputs; control-cluster Namespace label changes re-check provider-reference access. Rule changes re-resolve selected collections. It deliberately does **not** watch encryption Secrets, so their recovery is picked up by periodic reconciliation without retaining every Secret value in the control-plane cache. It watches the applier claims its `GitTarget`s name, for `ApplierClaimReady`. - `ApplierClaimReconciler` (one per applier claim kind) watches its applier claims, the `GitTarget`s naming - them, `GitProvider`s, `KubeProvider`s and `Namespace` labels. It does not watch appliers: it + them, `GitBranch`es, `GitProvider`s, `KubeProvider`s and `Namespace` labels. It does not watch appliers: it reads the Flux `Kustomization` or Argo CD `Application` on its tick to set `Configured`, so a cluster without the Flux or Argo CD CRDs needs no informer for them. -- `WatchRuleReconciler` / `ClusterWatchRuleReconciler` watch `GitTarget`, `GitProvider` and - `KubeProvider` (a `ClusterWatchRule` also `Namespace` labels), populate the RuleStore, and - trigger the rule-change reconcile. -- `GitProviderReconciler` validates reachability and manages the signing key lifecycle. +- `WatchRuleReconciler` / `ClusterWatchRuleReconciler` watch `GitTarget`, `KubeProvider` and + `Namespace` labels, plus the watch manager's stream-state events for `StreamsRunning`, populate + the RuleStore, and start the rule-change reconcile. +- `GitProviderReconciler` validates reachability and manages the signing key lifecycle. It watches + nothing but its own objects. +- `GitBranchReconciler` watches `GitProvider`s, `Namespace` labels and the `GitTarget`s naming it. + It proves the credential with a push session, keeps the branch claim, and publishes the worker's + `status.remote` (see [Git write architecture](#git-write-architecture)). +- `GitReceiverReconciler` watches only its own objects and publishes `status.webhookPath` once the + token `Secret` holds a token. - `KubeProviderReconciler` validates source-connection inputs, decides whether the provider may be used at all (the installation's `inCluster` consent, and which namespace holds its audit route), and projects its readiness to dependent targets. A provider created, deleted or moved to another route @@ -1778,9 +1829,9 @@ Controllers watch their dependencies so dependents reconcile quickly after spec otherwise, so the request claims no actor). Dependency watches use narrow predicates to avoid status-only heartbeat churn: a `KubeProvider` also -admits a `Ready` transition, and a `Namespace` admits only label changes. `GitProvider`, `GitTarget`, and -`CommitRequest` carry immutability constraints where a spec change would orphan a materialized subtree or -invalidate an in-flight finalize. +admits a `Ready` transition, and a `Namespace` admits only label changes. `GitProvider`, `GitBranch`, +`GitTarget`, and `CommitRequest` carry immutability constraints where a spec change would orphan a +materialized subtree or invalidate an in-flight finalize. *** @@ -1810,7 +1861,7 @@ flowchart TD Hq -->|no| J[Configured-author: no fact streams or index; audit webhook skipped] I --> K[Setup + register Watch Manager] J --> K - K --> L[Register GitProvider + KubeProvider + GitTarget + FluxClaim + ArgoCDClaim
+ CommitRequest controllers; admission webhooks] + K --> L[Register GitProvider + GitReceiver + KubeProvider + GitBranch + GitTarget
+ FluxClaim + ArgoCDClaim + CommitRequest controllers; admission webhooks] L --> M[Add cert watchers + health checks] M --> N[mgr.Start] ``` @@ -1832,104 +1883,59 @@ commit is committer-authored. ## Observability -- **Source**: [internal/telemetry/exporter.go](../internal/telemetry/exporter.go) - -Metrics are exported over OTLP / the metrics server. The reader's guide to every live family, with -copy-pasteable PromQL, is [interpreting-metrics.md](interpreting-metrics.md); the design behind the -current shape is [metrics-observability-plan.md](design/metrics-observability-plan.md). - -There are **two** ingestion paths, and the metrics follow that split. Watch carries object state and -decides whether an object reaches Git. Audit carries only the author's name: a missing or late fact -changes the author, never the state. They meet at the resolver and nowhere else. - -**Object state, stage by stage.** Every stage counts with a bounded `outcome`, so the pipeline reads -as one funnel and its loss paths as one selector: - -- **Ingest.** `gitopsreverser_watch_events_total{gittarget_*,group,version,resource,outcome}` counts - every delivered watch event exactly once, at `routeLiveTargetWatchEvent`, the single switch - carrying every terminal branch. `outcome` separates the pipeline working (`routed`, `unchanged`, - `bookmark`) from a refusal the watch delivers again (`route_failed`). Beside it, - `_watch_event_handling_seconds` (stream occupancy, the head-of-line signal), - `_watch_sessions_ended_total{reason}` and `_watch_replay_duration_seconds` (`410` pressure and - what a rebuild costs), `_watch_recovery_total{mode}`, and `_watch_types{state}`. -- **Queue.** `gitopsreverser_git_queue_refusals_total{kind}` counts work a worker refused at enqueue, - which its producer offers again, and `_git_queue_depth` is read at scrape time rather than published by the worker loop. -- **Write.** `gitopsreverser_git_documents_total{gittarget_*,group,version,resource,outcome}` is the - per-document census (`written`, `deleted_live`, `deleted_sweep`, `unchanged`, `retained`, `refused`), with - `_placements_total`, `_placement_refusals_total` and `_placement_kustomization_entries_total` - answering where a new document landed and which resources were refused. -- **Commit and push.** `gitopsreverser_git_commits_total{provider_*,branch,author_kind,message_source}` - counts commits that **reached the remote**, and `_git_pushes_total{outcome}`, - `_git_push_retries_total{reason}` and `_git_push_duration_seconds` cover the cycle that puts them - there. `author_kind="unresolved"` means attribution ran and could not name an actor. - `_git_fetches_total{reason}` is the other direction: it counts every call that runs a `SmartFetch` - (a push reads the remote's advertisement too, and is deliberately not counted here), - and it is the only way to answer "is the mirror still pulling on every write?", because the push - counters read identically whether the worker fetches constantly or never. On a healthy publishing - target `reason="publication"` is flat at zero. See - [Interpreting metrics](interpreting-metrics.md#reading-git_fetches_total). - -**Apply requests.** `apply_requests_total{applier_kind,reason,outcome}` counts every -attempt the worker decided to send (`applier_kind` is `kustomization` or `application`; `reason` is -`ForeignCommit`, `Refusal`, `Repair` or `CatchUp`), with `outcome` separating -`sent` from `failed` (the worker decides the retry), `dropped` (no target may still ask for that -reason) and `stale` (the tip moved before the patch). It counts attempts, not -the requests still due. `apply_echoes_total{outcome}` counts the applier's own writes the apply -window kept out of Git (`not_mirrored`) or mirrored because no window was open (`late`); -`apply_ownership_conflicts_total` counts objects another applier owns that were kept out of a -coordinated folder; and `stale_snapshots_total{outcome}` counts resync snapshots discarded because an -apply made them stale, or used after the 5-minute bound. -`receiver_deliveries_total{receiver_namespace,receiver_name,type,outcome}` counts receiver -deliveries per `GitReceiver`: `notified`, `unknown_repository` (no `GitProvider` has the repository), -`unknown_branch`, `ignored` (a ping, a tag, another event type), `rejected` -(bad signature) and `invalid`. -The receiver cannot tell a foreign push from an own one; the worker decides that from the -advertisement, which is why a `notified` delivery of the worker's own push is followed by no -request and only a foreign one by a `ForeignCommit`. See -[Interpreting metrics](interpreting-metrics.md#apply-requests-and-the-push-receiver). - -**Authorship**, when `--author-attribution` is on: - -- **Ingress.** `gitopsreverser_audit_eventlist_duration_seconds{outcome}` times every request at - `/audit-webhook`, rejections included (`bad_method`, `bad_path`, `bare_endpoint_disabled`), and - its `_count` series is the request counter. A named route is never rejected (it is a partition - name, not a claim about an object), so the only route-shaped rejection is the bare endpoint with - no annotation key configured. -- **Per-event census.** `gitopsreverser_audit_events_total{outcome,category,group,version,resource,verb}` - gives one terminal outcome per audit event; `category="error"` must stay zero. -- **Fact pipeline.** `_attribution_facts_total{op}`, `_attribution_fact_index_entries`, - `_attribution_fact_index_evictions_total{reason}`, `_attribution_fact_stream_gaps_total{stream}`, - `_attribution_fact_stream_decode_errors_total{transport}`, - `_attribution_fact_follower_errors_total` with `_..._last_success_timestamp_seconds`, and - `_attribution_transport_info{transport}`. The three loss counters are deliberately **not** one - metric: an eviction counts a fact, a trim gap counts an occurrence, and a decode error counts an - entry carrying a whole batch, so their sum would be in no unit at all. -- **The join.** `_attribution_resolutions_total{tier,actor_kind,group,version,resource}` says which - evidence named the author and who it named; `_resolution_wait_seconds{tier,event_kind}` says what - the grace wait cost, split by write and removal. - -**Discovery** (`api_catalog_*`) carries a `source_cluster` label (the provider's -`/`) and is recorded for every source cluster, so a degraded `APIService` on a -remote cluster is visible. **Secret encryption** is one counter, `_secret_encryptions_total{outcome}`. - -**Configuration state** is the one family that is not about flow. -`gitopsreverser_resource_condition{kind,resource_namespace,resource_name,type,status,reason}` -publishes the kstatus trio of every object a human declared (`GitTarget`, `WatchRule`, -`ClusterWatchRule`, `GitProvider`, `KubeProvider`, `FluxClaim`, `ArgoCDClaim`) as one series -per possible status, of which exactly one is `1`. It is recorded at `reconcileStatus.commit()`, the -single status choke point every controller but the `CommitRequest` one uses, and the object's -entry is dropped when the object goes: a condition series that outlives its object reports -`Ready=False` forever and never clears. It is the only family -carrying object names, which §7.1 of the plan permits for configuration objects and nothing else. - -Gauges here are **observable**: their value is read when Prometheus scrapes, not pushed from the -loop they measure, because a gauge published from inside a work loop reports the loop's last healthy -moment for as long as the loop is stuck. For the same reason "how long has this been waiting" is -exported as a timestamp (`watch_plan_oldest_dirty_since_timestamp_seconds`) and read as -`time() - `. - -The migration for the names that changed is in [UPGRADING.md](UPGRADING.md). -See [Operational Boundaries](#operational-boundaries). +- **Instruments**: [internal/telemetry/exporter.go](../internal/telemetry/exporter.go) +- **Reader's guide**: [interpreting-metrics.md](interpreting-metrics.md), the one place the live + instrument list, label vocabularies, queries and suggested alerts are maintained +- **Why the surface has this shape**: [metrics-observability-plan.md](design/metrics-observability-plan.md) + +Instruments are OpenTelemetry instruments, exported through the SDK's Prometheus exporter onto the +controller-runtime registry, so they are scraped from the metrics server's `/metrics` with every +controller-runtime series beside them. Nothing is pushed over OTLP. Every name starts with +`gitopsreverser_`. + +The families follow the architecture, which is why this section is a map rather than a list: + +| Family | Follows | In the guide | +|---|---|---| +| `resource_condition` | the kstatus trio of every object a human declares | [Configuration state](interpreting-metrics.md#configuration-state) | +| `watch_*` | ingest: events, sessions, replays, recoveries, resolved types, open streams | [Pipeline](interpreting-metrics.md#pipeline-watch-to-push) | +| `git_*`, `commit_requests_total` | the branch worker: documents, windows, commits, pushes, fetches, refusals, losses, the retained backlog | [Pipeline](interpreting-metrics.md#pipeline-watch-to-push), [Tuning commit windows](interpreting-metrics.md#tuning-commit-windows) | +| `placement*` | where a new document lands, and the ones refused | [New-file placement](interpreting-metrics.md#new-file-placement) | +| `apply_*`, `receiver_deliveries_total` | apply requests, the apply window and the push receiver | [Apply requests](interpreting-metrics.md#apply-requests-and-the-push-receiver) | +| `audit_*`, `attribution_*` | the optional author lookup: the door, the fact pipeline, the join | [Audit attribution](interpreting-metrics.md#audit-attribution-optional) | +| `api_catalog_*` | discovery, per source cluster | [API resource catalog](interpreting-metrics.md#api-resource-catalog) | +| `watch_plan_*` | the queue that plans each `GitTarget`'s watches | [Watch plane owner](interpreting-metrics.md#watch-plane-owner) | +| `secret_encryptions_total` | the path that keeps Secrets out of Git in plaintext | [Secret encryption](interpreting-metrics.md#secret-encryption) | + +Six rules hold across all of them, and a new instrument follows them too: + +- **The two ingestion paths are measured apart.** Watch carries object state and decides whether + an object reaches Git; audit carries only the author's name. They meet at the resolver and + nowhere else, so no metric adds one to the other. +- **Each stage counts once, with a bounded `outcome`.** The pipeline reads as a funnel, and its + loss paths (`git_commit_failures_total`, `git_writes_dropped_total`, + `placement_refusals_total`, `git_documents_total{outcome="refused"}`) as one selector. +- **Two identities, and one join between them.** Watch and document families are labeled by the + `GitTarget` (`gittarget_namespace`, `gittarget_name`); branch-worker families by the `GitBranch` + the worker writes for and its `GitProvider` (`gitbranch_*`, `provider_*`), because one worker + serves every `GitTarget` on it. `git_branch_targets` maps one to the other. `source_cluster` is + the `KubeProvider` as `/`. Keys are prefixed because a pod scrape with `honor_labels=false` renames a + bare `namespace` to `exported_namespace` and puts the scraped pod's own in its place. +- **Configuration objects are labels; mirrored resources never are.** A `GitTarget`, `GitBranch`, + applier claim or `GitReceiver` name is bounded by how many someone wrote, so it may be a label; a + mirrored resource is data and is never one, nor is a name a client supplies before it resolves. + A family that grows past its per-instrument cap (15000 points) collapses into one + `otel_metric_overflow="true"` series. +- **A gauge about a work loop is read at scrape time.** One published from inside the loop reports + its last healthy moment for as long as the loop is stuck, so the queue, backlog, watch-plane and + apply gauges are observable. "How long has this been waiting" is a timestamp, read as + `time() - `. The few recorded at the step they describe (`api_catalog_*`, the fact index + size, the attribution follower's last success, the transport) record a step that completed. +- **A metric needs an interpretation.** A new instrument lands with its recording site and its row + in the guide in the same change ([adding a metric](interpreting-metrics.md#adding-a-metric)). + +Renamed series are listed in [UPGRADING.md](UPGRADING.md). See +[Operational boundaries](#operational-boundaries) for what is not measured. *** diff --git a/docs/bi-directional-argocd.md b/docs/bi-directional-argocd.md index fa0c7941a..abc18d9d6 100644 --- a/docs/bi-directional-argocd.md +++ b/docs/bi-directional-argocd.md @@ -39,7 +39,7 @@ for each of these, and for the settings that look like a fix and are not. server-side apply under its own field manager, `gitops-reverser`, and touches nothing else of `automated`, so `prune` and `selfHeal` stay as they are. Switching automated sync on again (Argo CD's *Enable Auto-Sync*, `kubectl edit`) is undone within a minute, with a -`ClaimedResourceReconciled` Event, and counted in `claimed_resource_reconciled_total`; a sync Argo CD +`ClaimedResourceReconciled` Event, and counted in `apply_claimed_resource_reconciled_total`; a sync Argo CD started in that minute runs to its end, unchecked against the branch. Deleting the applier claim, or no longer naming it from any `GitTarget`, hands the switch back as it was: syncing on its own if it did before, manual if it was. The applier claim records which in diff --git a/docs/bi-directional-flux.md b/docs/bi-directional-flux.md index e070e3f75..e4dc209f5 100644 --- a/docs/bi-directional-flux.md +++ b/docs/bi-directional-flux.md @@ -478,7 +478,7 @@ tip, the `Kustomization`s GitOps Reverser holds ignore the new artifact, and one - **`flux resume ks checkout`** applies the `Kustomization` at once, without GitOps Reverser checking the commit first. Within a minute the applier claim suspends it again: it records a - `ClaimedResourceReconciled` Event, counts it in `claimed_resource_reconciled_total`, and shows the + `ClaimedResourceReconciled` Event, counts it in `apply_claimed_resource_reconciled_total`, and shows the apply as an `Unresolved` operation that completes once nothing it started is running. It does so every time. - **`flux reconcile ks checkout`** does nothing visible while the `Kustomization` is suspended: Flux @@ -534,7 +534,7 @@ flux get ks checkout -n flux-system ### Repeated `ClaimedResourceReconciled` events Repeated `ClaimedResourceReconciled` Events mean something keeps undoing the applier claim's -suspension. A rising `claimed_resource_reconciled_total` counts these corrections. A person who +suspension. A rising `apply_claimed_resource_reconciled_total` counts these corrections. A person who runs `flux resume` causes one each time, as [overriding by hand](#overriding-by-hand) describes. If they keep coming and the applier claim also reports `OperationUnresolved`, look at the controller that writes the `Kustomization`. diff --git a/docs/configuration.md b/docs/configuration.md index e9fe79247..d75d269a3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2216,8 +2216,9 @@ companions matter as much: - `gitopsreverser_placement_refusals_total{reason}` counts resources the operator **did not write**: a template that escapes `spec.path` (`invalid_path`), a sensitive resource whose path is already taken (`sensitive_append`), a plaintext resource routed at an encrypted file - (`plaintext_onto_encrypted`), and two resources of mixed sensitivity landing on one new file - (`mixed_sensitivity_new_file`). Any of these means a policy to fix; the write is retried after you + (`plaintext_onto_encrypted`), two resources of mixed sensitivity landing on one new file + (`mixed_sensitivity_new_file`), and a path whose multi-document file the writer cannot append to + without dropping a sibling (`multi_document_target`). Any of these means a policy to fix; the write is retried after you fix it. - `gitopsreverser_placement_kustomization_entries_total{outcome="failed"}` counts new files whose `resources:` entry could not be added. The file is committed and kustomize will never build it, so diff --git a/docs/design/metrics-observability-plan.md b/docs/design/metrics-observability-plan.md index 6361bb704..54fc721b5 100644 --- a/docs/design/metrics-observability-plan.md +++ b/docs/design/metrics-observability-plan.md @@ -595,8 +595,8 @@ Alerts, as rules rather than sketches this time: | Alert | Expression | Meaning | |---|---|---| -| Mirror stopped | `sum by (provider_namespace,provider_name,branch) (rate(gitopsreverser_git_pushes_total{outcome="failed"}[15m])) > 0 unless sum by (provider_namespace,provider_name,branch) (rate(gitopsreverser_git_pushes_total{outcome="pushed"}[15m])) > 0`, for 15m | this branch is failing to push AND landing nothing. The `unless` arm is required: a failed push retains its writes and is retried, so an occasional failure beside successful ones is contention, not an outage, and paging on any failure trains people to ignore it | -| Work refused on a healthy branch | `rate(gitopsreverser_git_queue_refusals_total[5m]) > 0 unless on(provider_namespace, provider_name, branch) gitopsreverser_git_intake_paused == 1` | the queue is too small for its bursts; producers offer the work again, so this is lag until a watch cursor expires | +| Mirror stopped | `sum by (gitbranch_namespace,gitbranch_name) (rate(gitopsreverser_git_pushes_total{outcome="failed"}[15m])) > 0 unless sum by (gitbranch_namespace,gitbranch_name) (rate(gitopsreverser_git_pushes_total{outcome="pushed"}[15m])) > 0`, for 15m | this branch is failing to push AND landing nothing. The `unless` arm is required: a failed push retains its writes and is retried, so an occasional failure beside successful ones is contention, not an outage, and paging on any failure trains people to ignore it | +| Work refused on a healthy branch | `rate(gitopsreverser_git_queue_refusals_total[5m]) > 0 unless on(gitbranch_namespace, gitbranch_name) gitopsreverser_git_intake_paused == 1` | the queue is too small for its bursts; producers offer the work again, so this is lag until a watch cursor expires | | Ingest loss | `rate(gitopsreverser_watch_events_total{outcome="route_failed"}[10m]) > 0` | observed changes are not reaching the writer. Often the same events as the row above, seen from the other end | | Audit misdirected | `rate(gitopsreverser_audit_eventlist_duration_seconds_count{outcome=~"bad_path\|bare_endpoint_disabled"}[15m]) > 0` | an apiserver is posting audit somewhere this operator will not read it (§2.6) | | Fact loss | `rate(gitopsreverser_attribution_fact_stream_gaps_total[10m]) > 0 or rate(gitopsreverser_attribution_fact_stream_decode_errors_total[10m]) > 0` | facts are gone for good. Two series in an `or`, never a sum: one counts occurrences and the other entries (§5.2) | diff --git a/docs/future/gitops-api-open-asks.md b/docs/future/gitops-api-open-asks.md index a6eb4671b..0d9927dec 100644 --- a/docs/future/gitops-api-open-asks.md +++ b/docs/future/gitops-api-open-asks.md @@ -7,7 +7,7 @@ left out: #30 shipped as `GitReceiver` (#432), #33 went with the prefetch in #42 `commit_requests_total`), #25 shipped in 0.44.0 (#343), and #6 is answered by immutability that fails at admission with "delete and recreate". Since revision 15: #32 is a row in [the Flux guide](../bi-directional-flux.md#troubleshooting); #31 is -`claimed_resource_reconciled_total`, labeled by applier kind and claim; ask #23 is pinned at the +`apply_claimed_resource_reconciled_total`, labeled by applier kind and claim; ask #23 is pinned at the fact index, for a finalizer cleared by a PATCH or an UPDATE, in either order (`TestFactIndex_AFinalizerClearedByAControllerNamesTheDeleter`); and #11 shipped in 0.41.0 (#290): create and update write through one encoder, `internal/yamlstyle`, and a test fails on a second diff --git a/docs/future/task-oriented-docs-and-website.md b/docs/future/task-oriented-docs-and-website.md new file mode 100644 index 000000000..b473480ef --- /dev/null +++ b/docs/future/task-oriented-docs-and-website.md @@ -0,0 +1,366 @@ +# Task-oriented documentation and a static website + +Status: proposed documentation and website plan. The navigation, page splits, and website described +here are future work. + +## Goal + +Give GitOps Reverser a public website that helps people understand the product, install it, and +complete common tasks. Use the same documentation sources for the repository and the website. + +Organize the reader-facing navigation around outcomes. A reader should be able to find the next +step without already knowing names such as `GitTarget`, attribution, or API-first publication. +Keep architecture and configuration reference available alongside the task guides. + +## The four main categories + +These categories form the documentation landing page and the website's documentation navigation. +They describe reader journeys; they do not require matching physical folders in the first phase. + +### 1. Get started + +Offer two visible setup paths. The deciding question is: does Flux or Argo CD apply the same folder +GitOps Reverser writes? If yes, follow the bi-directional path even when the immediate goal is only +to record cluster changes. An existing deployment loop needs coordination regardless of whether +anyone plans to edit Git manually. + +#### Capture cluster state in Git + +- **Install GitOps Reverser.** For a cluster administrator: prerequisites, credentials, permissions, + installation, a first captured change, verification, and cleanup. Keep the default path usable + without audit attribution. Start from the [quick start](../quickstart.md). +- **Start capturing a team's resources.** For a platform administrator or team owner: choose the + source cluster, namespaces, resource selection, repository, branch, and folder. Cover adopting + an existing repository through a scratch branch, supported layouts, deletion policy, and + cross-namespace access. Build a focused guide from [configuration](../configuration.md) and + the repository adoption guidance in the [project overview](../../README.md). +- **Record who made a change (optional).** For an administrator configuring authenticated Git + authorship: check cluster eligibility, configure audit delivery and attribution, and verify a + known actor's commit. Use the existing [attribution setup guide](../attribution-setup-guide.md) + as the home for this journey. See the dedicated scope below. + +This path reaches a verified commit without an applier claim or Git push webhook. Describe it as +capturing selected cluster state: batching and restart recovery mean Git history is not a complete +record of every event. Attribution is optional in both setup paths. + +#### Coordinate changes from Kubernetes and Git + +- **Choose bi-directional coordination.** Use [bi-directional GitOps](../bi-directional.md) as the + overview and entry point. Explain when the shared-folder deployment loop needs coordination, + then build on the installation and resource-selection steps above. +- **Configure the applier.** Follow the [Flux](../bi-directional-flux.md) or + [Argo CD](../bi-directional-argocd.md) setup guide to configure the claim and connect the target. + Explain who controls applying the folder and what happens when the operator is unavailable. +- **React promptly to Git pushes.** For an administrator enabling edits from both Git and + Kubernetes: configure the Git host's push webhook and a `GitReceiver`, then verify delivery and + the resulting apply. Make this an explicit step in bi-directional setup and link it from + "Make a change through Git." See the dedicated scope below. +- **Verify both directions.** Capture a Kubernetes edit, then push a Git edit and verify the + coordinated apply. Link to the everyday change guides and delayed-apply troubleshooting. + +#### Shared setup guidance + +Installation happens at the platform level; onboarding repeats for each team or destination. +Keep those outcomes separate so a team can start using an existing installation without repeating +the administrator's setup. + +Link supporting setup guides where readers need them: [GitHub](../github-setup-guide.md), +[Azure DevOps](../azure-devops-getting-started.md), [RBAC](../rbac.md), +[Secret encryption](../sops-age-guide.md), and [commit signing](../commit-signing.md). +Explain when each capability is required or optional. + +### 2. Make changes + +- **Make a change through Kubernetes.** For an end user: create, update, or delete a selected + resource; follow the change into Git; understand timing and refusal outcomes. Include a simple + worked example and links to deletion policy and supported manifest layouts. +- **Group changes and explain why.** For an end user: collect related edits, attach intent through + a `CommitRequest`, and verify the resulting commit or no-commit outcome. Explain author matching + and window timing. Build from the [commit-window demo](../demo/commit-window.md) and + [commit messages](../commit-messages.md). +- **Work alongside other people on the same branch.** For collaborating teams: explain how authors + and targets share a branch, what splits a commit window, and what happens when edits overlap. + Link to the shared conflict and publication contract in + [API-first publication](../api-first-publication.md). +- **Make a change through Git.** For an end user and the administrator enabling this workflow: + explain how an external push reaches the cluster through Flux or Argo CD, what happens to pending + Kubernetes edits, and how to verify the apply. Link back to the bi-directional setup prerequisites; + keep the common explanation in its overview and the everyday example in this task guide. + +Distinguish another person's Kubernetes edit from another person's Git push. Both can affect the +same branch, but they exercise different parts of the system and need different worked examples. +Link [installing applications as Kubernetes resources](../installing-apps-as-krm.md) as an expanded +example of choosing which declarations to capture. + +### 3. Operate + +- **Monitor health and resource usage.** For administrators: check readiness, publication progress, + latency, backlog, memory, watch count, and Git activity. Connect each observation to a question + and a next action. Use [interpreting metrics](../interpreting-metrics.md) as the shared reference. +- **Investigate a missing or unexpected change.** For administrators and end users: start from + symptoms such as no commit, an unresolved author, a refused folder, or a delayed apply. Identify + the relevant conditions, events, logs, and metrics. Explain recovery after an outage or restart, + including the limits of retained work and deletion recovery. +- **Upgrade an installation.** For administrators: identify migration requirements, apply the + upgrade, and verify the installation afterward. Keep release-specific changes in the + [upgrade guide](../UPGRADING.md); link to them from a stable upgrade procedure. + +Keep routine monitoring separate from symptom-driven troubleshooting. Readers investigating an +incident should reach a diagnostic path without first learning the entire metrics vocabulary. + +### 4. Understand and reference + +- **Architecture.** A concise account of the system: one Kubernetes edit reaching Git, one Git + edit reaching the cluster, component ownership, and safety and recovery boundaries. Refactor + [architecture](../architecture.md) around that reading path. +- **Configuration.** The resource relationships, settings, defaults, and examples. Keep + [configuration](../configuration.md) authoritative and link to it from task guides. +- **Security model.** Credentials, permissions, source identity, namespace boundaries, encryption, + and audit ingress trust. Use [the security model](../security-model.md) and the RBAC guide. +- **Supported manifest layouts.** What the writer can edit, what it refuses, how it preserves + existing documents, and how it places new ones. Provide a reader-facing explanation linked to + the [support contract](../design/support-boundary/support-contract.md) and configuration reference. +- **API, Helm, and command reference.** Exact fields, chart values, flags, and defaults. Establish + an explicit source for each surface and check it against the implementation. Start with + configuration and the [Helm chart reference](../../charts/gitops-reverser/README.md). +- **Glossary.** Extract the reader vocabulary from [definitions](../definitions.md) into a + published glossary. Keep maintainer naming rules in the existing document, linking to the + extracted definitions so each term still has one authoritative definition. Update citations + and anchors as part of that split. + +Keep [the component inventory](../components.md) and deeper implementation records accessible to +contributors through the existing [maintainer index](../INDEX.md). Rewrite the existing +[user landing page](../README.md) around the four categories. + +## Guide inventory and concrete backlog + +This is the initial mapping, completed as part of this plan. "Exists" means a usable source exists; +it still needs publication review. "Needs split" means extract or separate reader content from a +broader document. "Missing" means a dedicated task guide is absent, even where source material +already exists. Proposed filenames are decisions for implementation, not links to absent pages. + +| Guide or source | Category or destination | State and action | +| --- | --- | --- | +| `docs/README.md` | Documentation landing page | Exists: rewrite navigation | +| Root `README.md` | Homepage | Exists: reuse product explanation and demo | +| `alternatives.md` | Homepage: is this right for me? | Exists: link from adoption guidance | +| `quickstart.md` | Get started: capture cluster state | Exists: verify first commit without an applier or push webhook | +| Team onboarding | Get started: capture cluster state | Missing: extract selection, sharing, and repository adoption; check for an existing applier | +| `github-setup-guide.md` | Get started: GitHub credentials | Exists: link to push-notification setup | +| `azure-devops-getting-started.md` | Get started: Azure DevOps credentials | Exists: retain provider-specific setup | +| `attribution-setup-guide.md` | Get started: record who | Exists: retain outcome and eligibility opening | +| React promptly to Git pushes | Get started: bi-directional setup | Needs split: extract setup from `bi-directional.md`, add a GitHub walkthrough | +| `sops-age-guide.md` | Get started: encrypt sensitive resources | Exists: supporting guide | +| `commit-signing.md` | Get started: sign commits | Exists: supporting guide | +| Normal Kubernetes change | Make changes | Missing: expand the quick-start example into create, update, delete, and refusal outcomes | +| `demo/commit-window.md` | Make changes: group changes and explain why | Exists: use as the worked task guide | +| `commit-messages.md` | Understand and reference; linked from grouped changes | Exists: keep template reference separate from the task | +| Collaboration on a shared branch | Make changes | Missing: use publication semantics and multi-author examples | +| `api-first-publication.md` | Understand and reference; linked from collaboration | Exists: shared publication contract | +| `bi-directional.md` | Get started: bi-directional overview | Needs split: retain the setup decision and common behavior, extract receiver setup and everyday Git-edit example | +| `bi-directional-flux.md` | Get started: bi-directional Flux setup | Exists: link to receiver setup and verification of both directions | +| `bi-directional-argocd.md` | Get started: bi-directional Argo CD setup | Exists: link to receiver setup and verification of both directions | +| Make a change through Git | Make changes | Needs split: extract the everyday example from bi-directional guidance and link to setup prerequisites | +| `installing-apps-as-krm.md` | Make changes: capture application intent | Exists: expanded example | +| `interpreting-metrics.md` | Operate: monitor | Exists: shared metric reference and interpretation | +| Symptom-driven troubleshooting | Operate | Missing: gather diagnostic paths from configuration and metrics | +| Stable upgrade procedure | Operate | Missing: steps and verification linking to release changes | +| `UPGRADING.md` | Operate: release-specific migrations | Exists: retain release entries | +| `architecture.md` | Understand and reference | Needs split: overview, setup detail, and implementation rationale | +| `configuration.md` | Understand and reference | Needs split: extract onboarding steps, retain exact settings | +| `security-model.md` and `rbac.md` | Understand and reference; linked from setup | Exists: trust boundaries and access | +| Supported manifest layouts | Understand and reference | Needs split: reader guide from configuration and the support contract | +| `definitions.md` | Understand and reference: glossary | Needs split: publish vocabulary, retain maintainer naming rules | +| Chart `README.md` | Understand and reference: Helm | Exists: published reference sourced from the chart | +| API and command reference | Understand and reference | Needs split: consolidate existing field and flag descriptions, identify uncovered surfaces | +| `INDEX.md` and `components.md` | Maintainer entry point | Exists: retain repository links | +| `CONTRIBUTING.md`, `style-guide.md`, and e2e debugging | Maintainer entry point | Exists: retain repository links | + +## Push notifications as an explicit setup activity + +The outcome is "react promptly to Git pushes." Treat webhook setup as part of the normal +bi-directional installation path; capture-only installations can complete setup without it. +A Git-side commit leaves the cluster behind until the +coordinated apply runs; prompt notification shortens that interval and the opportunity for further +Kubernetes edits to overlap the incoming change. + +The applier's usual reconciliation schedule is insufficient here: an applier claim holds the +applier, and GitOps Reverser decides when it applies. A rejected push or periodic refresh still +detects remote changes when webhook delivery is unavailable. Describe these as fallback paths, +with their latency, and preserve the existing conflict semantics. Do not promise that webhooks +prevent all clashes or that mirroring stops working without them. + +Extract the setup from +[telling GitOps Reverser that somebody else pushed](../bi-directional.md#telling-gitops-reverser-that-somebody-else-pushed). +Keep [the GitReceiver reference](../configuration.md#gitreceiver) authoritative for fields and +delivery diagnostics. The task guide covers: + +1. Enable and expose the receiver listener at an HTTPS endpoint reachable by the Git host. +2. Create the token Secret and `GitReceiver`, then read `status.webhookPath`. +3. Configure a GitHub push webhook with that URL and secret. Verify current host instructions + when writing the walkthrough; link other supported host variants from the reference. +4. Follow the documented Flux or Argo CD rules for existing applier webhooks. Keep audit delivery + and Git push notifications clearly named as separate webhook purposes. +5. Push a known change and check host delivery, `GitBranch.status.lastPushNoticeAt`, the observed + remote commit, and the applier's result. A successful HTTP response alone does not prove apply. +6. Diagnose unreachable endpoints, rejected signatures, repository URL mismatches, and unmatched + branches. Cover token rotation, which changes the webhook path, and missed-delivery recovery. + +Link this guide from Git-host setup, both applier setup guides, and delayed-apply troubleshooting. + +## Attribution as a first-class optional journey + +The outcome is "record who made a change." Explain that the author comes from an authenticated +Kubernetes identity, which can represent a person or a service account. Explain the operator's +separate committer identity and the explicit unresolved-author outcome. + +Order the guide around these questions: + +1. What author information appears in Git with attribution enabled and disabled? Open with one + sentence describing the outcome. +2. Can this cluster deliver the required API-server audit events? Check eligibility before setup. +3. How are audit policy, delivery, ingress connectivity, certificates, source routing, and + attribution configured? Label settings by their API-server, Helm, CRD, or controller surface. +4. How does a known test change prove that delivery and matching work end to end? +5. How can an administrator diagnose missing evidence or an unresolved author? +6. How can optional OIDC claim mapping add a friendly display name and email? + +Retain the attribution setup guide's existing prerequisite: "managed control planes that hide +these settings cannot use this webhook." Its opening already states the outcome and the required +control over API-server flags and files. Preserve that precise boundary and the configured-author +installation path; verify any new provider-specific claims when writing them. + +Link this guide from normal changes and grouped changes. Keep "who made the change" and "why the +change was needed" as distinct tasks. Explain the separate admission-based submitter capture used +by `CommitRequest` where it affects matching, so readers do not assume audit setup controls both. + +## How the guides fit together + +Give each task guide a consistent structure: + +1. Goal and intended reader. +2. Prerequisites, permissions, and optional capabilities. +3. Steps with a concrete example. +4. Expected result and how to verify it. +5. Relevant failure cases and next actions. +6. Links to the shared explanation and exact reference. + +Create a separate document when a task has its own meaningful outcome. Keep closely related +variations together. Give each contract one authoritative home and link to it from the guides. + +Shorten the architecture overview by moving field inventories, OIDC setup instructions, and +implementation history to their appropriate guides or references. Bring manifest editing and its +support boundary forward. Qualify recovery guarantees at first mention: deletion policy matters, +unpublished work is held in memory, and high availability is unfinished. Preserve the architecture's +"optional but advised" wording for Redis and explain that planned HA requires it. Distinguish +current single-replica behavior from that future requirement. + +## Static website + +### Homepage and entry paths + +Create a product homepage with a short explanation of the problem GitOps Reverser solves, an +API-to-Git diagram, and an example change with its resulting commit. Give visitors two prominent +actions: "Install GitOps Reverser" and "Understand how it works." + +Show the supported workflows and important adoption boundaries before installation. Include a +visible link to attribution eligibility. Present "Capture cluster state in Git" and "Coordinate +changes from Kubernetes and Git" as distinct setup paths under Get started, with the shared-folder +decision visible before readers choose. Keep the four top-level categories. +Use [alternatives](../alternatives.md) for the "is this right for me?" decision. Reuse existing demos +and diagrams where they explain the behavior clearly. + +The documentation landing page exposes the four categories above. Add search, page-local contents, +and related next steps. Keep source code, releases, contribution guidance, and issue reporting +easy to find without placing maintainer plans in the main user navigation. + +### Content and publishing requirements + +- Build static pages from repository Markdown with one maintained source per guide. Keep the + source readable on GitHub and avoid a separately maintained website copy of the documentation. +- Define a curated publication manifest. Exclude `spec/`, `design/`, `facts/`, `finished/`, + `layout/`, `future/`, `review/`, `local-only/`, and `*.ignore.md` from publication and search by + default. Publish extracted reader guides deliberately; link remaining maintainer references to + their repository pages. Only explicitly listed pages enter the build. +- Require every published Markdown source to appear in [.docs-lint-scope](../../.docs-lint-scope). + Check this inclusion automatically in CI; the prose scope may also contain unpublished contracts. + Add pages to the gate as they are cleaned, before publishing them. +- Require Mermaid rendering and existing image support. Preserve relative `.md` links in source + so GitHub navigation works; resolve published targets to site routes and unpublished references + to the corresponding repository revision. +- Preserve stable page URLs and heading anchors. Record redirects when pages move or split, and + retain valid repository links and source-code citations. +- Generate search only from the publication manifest and check the installation path on mobile + and with keyboard navigation. +- Label which release the installation instructions and reference describe. Keep development docs + identifiable; decide whether archived release versions are needed before adding a version selector. +- Provide a reproducible local build, a preview for review, and automated checks for generated + links, assets, and site output before publication. + +Choose the static site generator, hosting service, domain, and release publication policy during +implementation. Evaluate them against these requirements and maintenance cost. The content plan +does not depend on a particular framework or hosting provider. + +## Implementation phases + +### Phase 1: Apply the inventory to navigation + +- Use the inventory above to rewrite `docs/README.md` under the four categories, with both setup + paths under Get started and `bi-directional.md` as the coordination entry point. +- Retain `docs/INDEX.md` as the maintainer entry point and link it explicitly. +- Give every inventory row a destination or a named extraction task; avoid empty public pages. +- Preserve the existing maintainer lifecycle folders and source citations. Decide physical moves + only when they improve maintenance; navigation can provide the new structure immediately. + +### Phase 2: Complete the essential reading paths + +- Establish capture-only installation through first verified commit as the shortest complete path. +- Fill the missing onboarding, normal-change, collaboration, troubleshooting, and stable upgrade + guides identified above. +- Complete the bi-directional path: applier setup, push notifications with a GitHub example, and + verification of both directions. Link everyday Git edits back to these prerequisites. +- Complete the marked splits and attribution ordering, checking examples against supported behavior. + +### Phase 3: Build the website + +- Select the generator and hosting, then implement the homepage and curated documentation build. +- Add navigation, search, diagram rendering, link resolution, release labeling, and previews. +- Validate the essential reading paths in a rendered preview, including mobile and keyboard use. +- Configure the chosen domain and publication workflow, then publish the reviewed site. + +### Phase 4: Maintain it with the product + +- Update the affected task guide and shared contract when behavior changes. +- Validate installation examples against the release they document. +- Check source and rendered links, preserve redirects, and keep the publication list explicit. +- Use installation feedback and support questions to improve the task paths. + +## Completion criteria + +- Every inventory row has a published destination or an intentional maintainer-only destination. +- The homepage links directly to installation and architecture; the documentation landing page + presents all four categories with working task links and both setup paths under Get started. +- A recorded walkthrough against the documented release reaches a first commit from installation, + without an applier claim or Git push webhook, then onboards a second target using the existing + installation. Each guide names the expected observable result. +- Onboarding asks whether an applier deploys the same folder and routes that case to bi-directional + setup. The Flux and Argo CD paths each verify a Kubernetes edit reaching Git and a Git edit + reaching the cluster. The everyday Git-edit guide links back to those prerequisites. +- The attribution page states its outcome and eligibility before commands, retains the + configured-author path, and includes a known-actor verification example. +- The GitHub receiver walkthrough verifies host delivery, branch notification, remote observation, + and apply outcome. Both applier guides link to it and document fallback detection. +- Worked examples cover a Kubernetes edit, a Git push, and grouped changes with intent. + Troubleshooting includes no commit, unresolved author, refused folder, and delayed apply. +- CI rejects a published Markdown source absent from `.docs-lint-scope`, an unlisted built page, + or a broken rendered link or asset. Excluded maintainer pages do not enter the search index. +- The preview renders the existing Mermaid diagrams and preserves headings and relative links; + changed routes have tested redirects. Installation works on mobile and with keyboard navigation. +- `task lint-docs`, Vale on touched prose, and the site build checks pass. Website configuration + changes also pass the applicable repository validation gates. + +## Change isolation + +The plan was drafted on `docs-metrics-architecture-inventory`. Carry it and the documentation work +in a separate documentation PR from the metrics changes. Preserve unrelated work and avoid a +branch-wide commit or file move as part of the navigation rewrite. diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index 5099d583f..7a03c6658 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -1,8 +1,9 @@ -# Interpreting GitOps Reverser Metrics +# Interpreting GitOps Reverser metrics -> Last updated: September 2026, reconciled to the live instrument set in -> [`internal/telemetry/exporter.go`](../internal/telemetry/exporter.go). Names changed in that pass; -> [`UPGRADING.md`](UPGRADING.md) carries the old-to-new table. +> Last reconciled: October 2026, against the instruments in +> [`internal/telemetry/exporter.go`](../internal/telemetry/exporter.go) and their recording sites: +> every name, label key and label value below is one the code emits. +> [`UPGRADING.md`](UPGRADING.md) carries the old-to-new table for renamed series. The operator's field guide to the metrics GitOps Reverser exports: what each family means, and copy-pasteable PromQL for the questions operators actually ask. Every metric here has a production @@ -24,7 +25,7 @@ Three instrument shapes appear: | Shape | Suffix | How to read it | | --- | --- | --- | -| **Counter** | `_total` | Monotonic. Always wrap in `rate(...[5m])` or `increase(...[1h])` — the raw value is meaningless. | +| **Counter** | `_total` | Monotonic. Always wrap in `rate(...[5m])` or `increase(...[1h])`; the raw value is meaningless. | | **Histogram** | `_seconds` | Exposes `_bucket`, `_sum`, `_count`. Use `histogram_quantile()` for percentiles; `_count` is a free counter of observations. | | **Gauge** | (none) | Instantaneous value; read directly. | @@ -32,9 +33,9 @@ Three instrument shapes appear: A histogram named `foo_seconds` produces three series: -- `foo_seconds_bucket{le="..."}` — cumulative count per bucket boundary -- `foo_seconds_count` — total number of observations (use it like a counter) -- `foo_seconds_sum` — sum of all observed values +- `foo_seconds_bucket{le="..."}`: cumulative count per bucket boundary +- `foo_seconds_count`: total number of observations (use it like a counter) +- `foo_seconds_sum`: sum of all observed values Percentile: @@ -86,18 +87,18 @@ without a kubeconfig. - `type` is the kstatus trio: `Ready` (the summary), `Reconciling` (working on it) and `Stalled` (given up, and nothing will retry without a human). - `status` is `True`, `False` or `Unknown`. -- `reason` is the condition's reason as it stands — `ValidationFailed`, `UnsupportedContent`, - `BranchConflict` — which is what turns a count panel into a first guess. It is the API's own +- `reason` is the condition's reason as it stands (`ValidationFailed`, `UnsupportedContent`, + `BranchConflict`), which is what turns a count panel into a first guess. It is the API's own string, so a release that renames a reason changes the series a selector matches; [`UPGRADING.md`](UPGRADING.md) lists every rename. **Each condition publishes three series, one per possible status, of which exactly one is `1`.** So every query below compares `== 1`; a selector without it matches the two zeroes as well. The -alternative shape — one series whose *value* encodes the status — cannot be selected on and cannot +alternative shape (one series whose *value* encodes the status) cannot be selected on and cannot be aggregated, and `count by (status)` over it means nothing. **`reason` is part of the series identity.** A condition that changes reason changes series, so any -aggregation *over time* has to remove it first — otherwise a transition looks like one series ending +aggregation *over time* has to remove it first; otherwise a transition looks like one series ending and another starting: ```promql @@ -109,16 +110,16 @@ avg_over_time(sum without (reason) ( state, and an absent series is indistinguishable from an operator that is not running. **A deleted object stops publishing.** The series ends when the object goes, rather than reporting -its last value forever — which matters more than it sounds: a condition gauge that keeps a deleted +its last value forever. That matters more than it sounds: a condition gauge that keeps a deleted object at `Ready=False` manufactures a permanent false positive, and a panel that is permanently red is a panel nobody reads. -This is the only family carrying object names, and -[metrics-observability-plan.md](design/metrics-observability-plan.md) §7.1 permits it here and -nowhere else. These objects are configuration — bounded by how many someone wrote — where the -resources being mirrored are data, unbounded, and never a label. `CommitRequest` is excluded on the -same reasoning: one is created per save, so it is data-shaped, and its state is read from -`commit_requests_total` in aggregate and from the object's own conditions individually. +Object names are labels here because these objects are configuration, bounded by how many someone +wrote, as are the `GitTarget`, `GitBranch`, applier claim and `GitReceiver` names other families +carry. The resources being mirrored are data, unbounded, and never a label +([metrics-observability-plan.md](design/metrics-observability-plan.md) §7.1). `CommitRequest` is +excluded on the same reasoning: one is created per save, so it is data-shaped, and its state is +read from `commit_requests_total` in aggregate and from the object's own conditions individually. **Which configuration objects are not Ready?** The panel to start from: @@ -126,7 +127,7 @@ same reasoning: one is created per save, so it is data-shaped, and its state is gitopsreverser_resource_condition{type="Ready", status="False"} == 1 ``` -**The fleet, in one panel** — how many objects of each kind are in each state: +**The fleet, in one panel**: how many objects of each kind are in each state: ```promql count by (kind, status) (gitopsreverser_resource_condition{type="Ready"} == 1) @@ -158,11 +159,11 @@ count by (kind, reason) ( The shape is `gotk_reconcile_condition`'s, deliberately: the geometry is battle-tested and the Grafana panels for it already exist. The name is not, because the GitOps Toolkit is defined by -membership under the `fluxcd` organization rather than by behaviour, so `gotk_` on our series would +membership under the `fluxcd` organization rather than by behavior, so `gotk_` on our series would be a false statement about where they came from no matter how closely the conditions match. For anyone who wants these objects inside Flux-shaped panels, the alias belongs on the consumer's -side, as a recording rule — opt-in, committing us to nobody else's schema: +side, as a recording rule: opt-in, committing us to nobody else's schema: ```yaml # Optional. Surfaces gitops-reverser config objects in GOTK-shaped dashboards. @@ -190,36 +191,36 @@ boundary, the commit, the push. Background: | Metric | Type | Labels | Notes | | --- | --- | --- | --- | | `watch_events_total` | counter | `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource`, `outcome` | The ingest census: every delivered watch event, exactly once. `outcome` is `routed` / `unchanged` / `not_object` / `bookmark` / `shutdown` / `stream_error` / `route_failed`. | -| `watch_event_handling_seconds` | histogram | `group`, `version`, `resource` | How long a stream was **busy** on one event, attribution wait included. Occupancy, not queue delay. | -| `watch_sessions_ended_total` | counter | `group`, `version`, `resource`, `reason` | `closed` (the API server's routine watch timeout: the most common ending on a healthy cluster) / `expired` (cursor out of history, forcing a rebuild) / `error` / `stopped`. | -| `watch_replay_duration_seconds` | histogram | `group`, `version`, `resource` | Time to `initial-events-end`: what a `410` storm charges. | +| `watch_event_handling_seconds` | histogram | `source_cluster`, `group`, `version`, `resource` | How long a stream was **busy** on one event, attribution wait included. Occupancy, not queue delay. | +| `watch_sessions_ended_total` | counter | `source_cluster`, `group`, `version`, `resource`, `reason` | `closed` (the API server's routine watch timeout: the most common ending on a healthy cluster) / `expired` (cursor out of history, forcing a rebuild) / `error` / `stopped`. | +| `watch_replay_duration_seconds` | histogram | `source_cluster`, `group`, `version`, `resource` | Time to `initial-events-end`: what a `410` storm charges. | | `watch_recovery_total` | counter | `gittarget_namespace`, `gittarget_name`, `group`, `resource`, `mode` | One per completed recovery. `mode` is `cursor_resume` / `type_reconcile` / `replay` / `list_fallback`. No `version`: a recovery covers a resource collection. | | `watch_types` | gauge | `source_cluster`, `gittarget_namespace`, `gittarget_name`, `state` | Types this target resolves, by `streaming` / `replaying` / `blocked`. `sum` is the resolved total. | | `watch_streams_open` | gauge | `source_cluster`, `gittarget_namespace`, `gittarget_name` | Watch **sessions** currently open against the source cluster. Not the same as types: one type across three namespaces is three sessions. | | `git_documents_total` | counter | `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource`, `outcome` | The write-boundary census, per **document**. `outcome` is `written` / `deleted_live` / `deleted_sweep` / `unchanged` / `retained` / `refused`. | -| `git_commits_total` | counter | `provider_namespace`, `provider_name`, `branch`, `author_kind`, `message_source` | Commit batches that **reached the remote**. | -| `git_pushes_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per push cycle: `pushed` or `failed`. | -| `git_push_retries_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Replay rounds inside a cycle. `reason` is `remote_moved`. | -| `git_push_duration_seconds` | histogram | `provider_namespace`, `provider_name`, `branch` | One cycle end to end, retries included. | -| `git_fetches_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Every call that reads the remote through a `SmartFetch`. `reason` is `bootstrap` (repository preparation; **no production caller reaches it today**, so the series stays at zero) / `publication` (the head of a publication cycle) / `recovery` (re-establishing a base that could not be trusted, including a worktree a failed write left dirty that could not be undone locally) / `contention` (the reset onto the new tip after a push was rejected because somebody else moved the branch) / `push_failure_probe` (a push that failed without the remote saying anything, looking up where the branch is) / `forced_recheck` (a full re-read outside the publication cycle: a forced recheck, or the snapshot a resync judges against) / `refresh` (the periodic top-up of an idle branch, or a push receiver delivery on one, and only when the branch had actually moved). See the note below. | -| `git_queue_refusals_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind` | Work refused at enqueue: the queue was full, intake was paused through an outage, or the worker was stopping. `kind` is `write` / `attach` / `resync` / `refresh`. The producer keeps a refused item and offers it again: the watch delivers a write again from its cursor, the controller re-sends a save, a resync is gathered again, and the next reconcile asks for a refresh. What it can cost, when its watch cursor expires before the write comes back, is the write's own commit and, under `prune.mode: OnEvent`, a deleted object's removal from Git. | -| `git_commit_failures_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind`, `reason` | A window or request that died between routing and pushing. `kind` is `window` / `atomic`; `reason` is `refused` (a Git path a human must fix) / `error`. Every increment is a window's events lost until the next resync. An unreachable remote does not count: a decided window waits for the publication retry instead. | -| `git_writes_dropped_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind`, `reason` | A live event (`kind="event"`) or atomic write (`kind="atomic"`) the branch worker dropped because its `GitTarget`'s render fidelity was not established. `reason` is `render_fidelity_diverged`: a rendered value does not match the live object, and every write is dropped until a fresh replay measures the folder clean. Or it is `render_fidelity_pending`: a collection of the `GitTarget` has not finished its replay, which closes every collection of that target until it does. A live event that meets a pending gate is held, not dropped, and committed with its author once the gate opens; it is counted here only when the hold gives up, 10 minutes after the first one was held or once the branch holds more than its retained-byte budget, and the worker then asks for a fresh snapshot, which commits the current state as the operator (a deletion comes back only under prune mode `Always`). An atomic write is dropped at once. Nothing replays a dropped write, so each one is an edit missing from Git until the object changes again or its collection replays. The worker logs the first drop under each gate status at Info, naming the collections the gate waits for, and each hold, release and give-up at Info. | -| `git_commit_windows_total` | counter | `gittarget_namespace`, `gittarget_name`, `close_reason`, `timer_source` | One per closed commit window, whether or not it then produced a commit. `close_reason` is `idle_timeout` / `max_duration` / `attach_next` / `identity_change` / `buffer_limit` / `resync_before_apply` / `atomic_before_apply` / `foreign_commit` (a Git host said the branch may have moved, so the window is published now) / `apply_due` (an apply came due for an applier on the branch, so the window's edits are published before the apply is sent) / `shutdown`; `timer_source` is `target` or `commit_request`. See [tuning commit windows](#tuning-commit-windows). | +| `git_commits_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `author_kind`, `message_source` | Commit batches that **reached the remote**. | +| `git_pushes_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per push cycle: `pushed` or `failed`. | +| `git_push_retries_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `reason` | Replay rounds inside a cycle. `reason` is `remote_moved`. | +| `git_push_duration_seconds` | histogram | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | One cycle end to end, retries included. | +| `git_fetches_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `reason` | Every call that reads the remote through a `SmartFetch`. `reason` is `publication` (the head of a publication cycle, and a worker's first contact with the repository) / `recovery` (re-establishing a base that could not be trusted, including a worktree a failed write left dirty that could not be undone locally) / `contention` (the reset onto the new tip after a push was rejected because somebody else moved the branch) / `push_failure_probe` (a push that failed without the remote saying anything, looking up where the branch is) / `forced_recheck` (a full re-read outside the publication cycle: a forced recheck, or the snapshot a resync judges against) / `refresh` (the periodic top-up of an idle branch, or a push receiver delivery on one, and only when the branch had actually moved). See the note below. | +| `git_queue_refusals_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `kind` | Work refused at enqueue: the queue was full, intake was paused through an outage, or the worker was stopping. `kind` is `write` / `attach` / `resync` / `refresh`. The producer keeps a refused item and offers it again: the watch delivers a write again from its cursor, the controller re-sends a save, a resync is gathered again, and the next reconcile asks for a refresh. What it can cost, when its watch cursor expires before the write comes back, is the write's own commit and, under `prune.mode: OnEvent`, a deleted object's removal from Git. | +| `git_commit_failures_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `reason` | A live commit window that died between routing and pushing; a resync's failure is `git_resync_failures_total`. `reason` is `refused` (a Git path a human must fix) / `error`. Every increment is a window's events lost until the next resync. An unreachable remote does not count: a decided window waits for the publication retry instead. | +| `git_writes_dropped_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `reason` | A live event the branch worker dropped because its `GitTarget`'s render fidelity was not established. `reason` is `render_fidelity_diverged`: a rendered value does not match the live object, and every write is dropped until a fresh replay measures the folder clean. Or it is `render_fidelity_pending`: a collection of the `GitTarget` has not finished its replay, which closes every collection of that target until it does. A live event that meets a pending gate is held, not dropped, and committed with its author once the gate opens; it is counted here only when the hold gives up, 10 minutes after the first one was held or once the branch holds more than its retained-byte budget, and the worker then asks for a fresh snapshot, which commits the current state as the operator (a deletion comes back only under prune mode `Always`). Nothing replays a dropped write, so each one is an edit missing from Git until the object changes again or its collection replays. The worker logs the first drop under each gate status at Info, naming the collections the gate waits for, and each hold, release and give-up at Info. | +| `git_commit_windows_total` | counter | `gittarget_namespace`, `gittarget_name`, `close_reason`, `timer_source` | One per closed commit window, whether or not it then produced a commit. `close_reason` is `idle_timeout` / `max_duration` / `attach_next` / `identity_change` / `buffer_limit` / `resync_before_apply` / `foreign_commit` (a Git host said the branch may have moved, so the window is published now) / `apply_due` (an apply came due for an applier on the branch, so the window's edits are published before the apply is sent) / `shutdown`; `timer_source` is `target` or `commit_request`. See [tuning commit windows](#tuning-commit-windows). | | `git_commit_window_duration_seconds` | histogram | `gittarget_namespace`, `gittarget_name` | How long each window collected: from the write that opened it to the start of its finalize. Its count equals `git_commit_windows_total` summed over `close_reason` and `timer_source`. | -| `git_queue_depth` | gauge | `provider_namespace`, `provider_name`, `branch` | Pending + in-flight + committed-but-unpushed. Read at scrape time. | -| `git_retained_bytes` | gauge | `provider_namespace`, `provider_name`, `branch` | What the worker holds until a push lands, in the units of the retained-byte budget: the open window, the retained writes and waiting saves, each with a 1 KiB per-item charge. Serialized bytes, not heap. | -| `git_retained_writes` | gauge | `provider_namespace`, `provider_name`, `branch` | Decided writes waiting for a push. | -| `git_intake_paused` | gauge | `provider_namespace`, `provider_name`, `branch` | 1 while the worker refuses new writes, saves and resyncs because an outage filled its budget; 0 otherwise. | -| `git_oldest_retained_write_timestamp_seconds` | gauge | `provider_namespace`, `provider_name`, `branch` | When the oldest write still waiting for a push was decided, as a Unix timestamp. No sample while nothing waits. | -| `git_next_retry_timestamp_seconds` | gauge | `provider_namespace`, `provider_name`, `branch` | When the next attempt at the pending writes is due, as a Unix timestamp. No sample while no retry is scheduled. | -| `git_materialization_failures_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Attempts to commit decided writes that stopped before any push: `reason` is `unreachable` or `parent_unavailable`. Nothing is lost; the writes wait for the retry. A failure `git_pushes_total` cannot see, because no push cycle started. | -| `git_branch_targets` | gauge | `provider_namespace`, `provider_name`, `branch`, `gittarget_namespace`, `gittarget_name`, `source_cluster` | Always 1. The **join** between the GitTarget-labeled half of the pipeline and the branch-labeled half. Published per configured GitTarget, whether or not its worker runs. | +| `git_queue_depth` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | Pending + in-flight + committed-but-unpushed. Read at scrape time. | +| `git_retained_bytes` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | What the worker holds until a push lands, in the units of the retained-byte budget: the open window, the retained writes and waiting saves, each with a 1 KiB per-item charge. Serialized bytes, not heap. | +| `git_retained_writes` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | Decided writes waiting for a push. | +| `git_intake_paused` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | 1 while the worker refuses new writes, saves and resyncs because an outage filled its budget; 0 otherwise. | +| `git_oldest_retained_write_timestamp_seconds` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | When the oldest write still waiting for a push was decided, as a Unix timestamp. No sample while nothing waits. | +| `git_next_retry_timestamp_seconds` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | When the next attempt at the pending writes is due, as a Unix timestamp. No sample while no retry is scheduled. | +| `git_materialization_failures_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `reason` | Attempts to commit decided writes that stopped before any push: `reason` is `unreachable` or `parent_unavailable`. Nothing is lost; the writes wait for the retry. A failure `git_pushes_total` cannot see, because no push cycle started. | +| `git_branch_targets` | gauge | `gitbranch_namespace`, `gitbranch_name`, `gittarget_namespace`, `gittarget_name`, `source_cluster` | Always 1. The **join** between the GitTarget-labeled half of the pipeline and the GitBranch-labeled half: the `GitTarget`'s `spec.gitBranchRef`. Published per configured GitTarget, whether or not its GitBranch exists or its worker runs. | | `commit_requests_total` | counter | `outcome`, `gittarget_namespace`, `gittarget_name` | One per `CommitRequest` terminal **decision**. `outcome` is `committed` / `no_window` / `window_mismatch` / `already_present` / `failed`. A target that never resolved publishes the empty pair. See the counting note below. | | `placements_total` | counter | `source`, `disposition`, `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource` | One per new document at a resolved path. | | `placement_refusals_total` | counter | `reason`, `gittarget_namespace`, `gittarget_name`, `group`, `version`, `resource` | One per new resource the writer declined. Every increment is a resource **absent** from the mirror. | | `placement_kustomization_entries_total` | counter | `outcome`, `gittarget_namespace`, `gittarget_name` | `added` / `no_change` / `failed`. | -| `git_resync_failures_total` | counter | `gittarget_namespace`, `gittarget_name` | Rule-change resyncs whose apply failed **after** enqueue. | +| `git_resync_failures_total` | counter | `gittarget_namespace`, `gittarget_name`, `reason` | Rule-change resyncs that did not apply. `reason` is `error` (the commit failed) or `timeout` (no answer in time): the snapshot did not land. `refused` (refused at intake; the same refusal is `git_queue_refusals_total{kind="resync"}`, and the stream gathers it again) and `deferred` (the remote could not be reached; it stays pending and applies later, beside `git_materialization_failures_total`) lose nothing. A resync's commit failure is counted here only, never in `git_commit_failures_total`. | `source_cluster` is the `KubeProvider` the `GitTarget` mirrors from, as `/` (`gitops-reverser/local` for the chart's provider), so two providers with one name in two namespaces @@ -243,7 +244,7 @@ flatten the metric precisely where a push problem is being investigated. **`contention` and `push_failure_probe` are separate because the two failures mean different things.** A rejected push already carries the remote's hash, so it costs exactly one fetch: the reset onto the new tip, counted as `contention`. A push that failed before the remote said -anything — an auth failure, a dropped connection — has to look the hash up first, and that lookup +anything (an auth failure, a dropped connection) has to look the hash up first, and that lookup is the probe. Most probes find the remote unmoved, and then no reset follows and the probe is the only fetch the failure cost. Folding the two together would make an expired credential read as other writers fighting you over the branch. @@ -275,7 +276,7 @@ parent change, or a replay that did not finish. `contention` tracks `git_push_retries_total` one-for-one, because both count a confirmed moved remote. `push_failure_probe` moving on its own is pushes failing for a -reason that is not another writer — check credentials and connectivity, not the branch. +reason that is not another writer: check credentials and connectivity, not the branch. `forced_recheck` moving with nothing else is reconcile traffic. `refresh` is the price of `--git-refresh-interval`, per branch. Multiply its per-second rate by @@ -320,14 +321,14 @@ did. A workload that sits comfortably at a fraction of the depth can still drop administrative one. **Size it by burst, not by rate.** A queue slot holds one write request, so what has to fit is the -number of concurrent write requests a bounded burst can produce — roughly +number of concurrent write requests a bounded burst can produce, roughly `(concurrent writers) × (GitTargets sharing the branch worker)`. A burst that cannot exceed the depth is never refused, whatever its arrival shape. Note the second factor: workers are keyed by repository branch, so pointing a second GitTarget at a branch, from any namespace, halves the -headroom of a worker already in use, and nothing in the API surface says so. The branch series keep -their `provider_namespace` and `provider_name` labels: they name the one `GitProvider` that owns the -branch. Confirm -which targets share one worker by joining on `git_branch_targets` — several targets against a +headroom of a worker already in use. The branch series are labeled by the `GitBranch` the worker +writes for and its `GitProvider`, and `GitBranch.status.gitTargetCount` says how many targets +reference it. Confirm +which targets share one worker by joining on `git_branch_targets`: several targets against a single `git_queue_depth` series is one worker serving all of them. **What it costs.** On a healthy branch `--branch-buffer-max-size` does **not** cover this queue. That @@ -339,7 +340,7 @@ as well; see the backpressure paragraph below. Budget it as **queue depth × serialized size × ~6**. A queued object is held as an unstructured map, which measures around six times the bytes it serializes to; the channel itself is negligible beside that. So a full queue of 1000 at the ~800-byte resources of a small CRD is about 5 MiB per -saturated worker, against a 1Gi default limit — but the multiplier is on the payload, so a folder +saturated worker, against a 1Gi default limit, but the multiplier is on the payload, so a folder mirroring megabyte-sized ConfigMaps or Secrets reaches gigabytes at the same depth. Size against your own payloads, and against how many workers could saturate at once: the figure is per worker, and there is one worker per repository branch. @@ -376,9 +377,9 @@ every `GitTarget` on the branch says so on `Ready`. A healthy branch never refus `placement_refusals_total` and `git_documents_total{outcome="refused"}` **overlap the same way**: a resource the writer declines to place increments both, so the loss class holds two views of one -refusal rather than two refusals. Read them for different questions — `placement_refusals_total` +refusal rather than two refusals. Read them for different questions (`placement_refusals_total` carries the `reason` and fires only on the new-file path, `git_documents_total{outcome="refused"}` is -the write-boundary census view — and never add them. They are not quite 1:1 either: a refusal raised +the write-boundary census view) and never add them. They are not quite 1:1 either: a refusal raised after placement has already resolved reaches the census without a reason, so the census total is the one to trust for "how many documents were refused". @@ -398,22 +399,22 @@ sum(rate(gitopsreverser_git_pushes_total{outcome="pushed"}[5m])) successes is contention. A branch failing with nothing landing is an outage: ```promql -sum by (provider_namespace, provider_name, branch) ( +sum by (gitbranch_namespace, gitbranch_name) ( rate(gitopsreverser_git_pushes_total{outcome="failed"}[15m])) unless -sum by (provider_namespace, provider_name, branch) ( +sum by (gitbranch_namespace, gitbranch_name) ( rate(gitopsreverser_git_pushes_total{outcome="pushed"}[15m])) > 0 ``` **Which GitTargets does that branch serve?** The push-side instruments are labeled by -`{provider_namespace, provider_name, branch}` because one BranchWorker serves a (GitProvider, -branch) that several GitTargets can share. `git_branch_targets` is the join that names them: +`{gitbranch_namespace, gitbranch_name}` because one BranchWorker serves one `GitBranch`, which +several GitTargets can share. `git_branch_targets` is the join that names them: ```promql -sum by (provider_namespace, provider_name, branch) ( +sum by (gitbranch_namespace, gitbranch_name) ( rate(gitopsreverser_git_pushes_total{outcome="failed"}[5m]) ) - * on(provider_namespace, provider_name, branch) + * on(gitbranch_namespace, gitbranch_name) group_right gitopsreverser_git_branch_targets ``` @@ -425,19 +426,41 @@ one GitTarget uses the branch and fails the moment a second one does. **Read the result as "potentially affected", never as "behind".** The join says which GitTargets write to a branch and nothing about whether any of them had pending work. Two limits follow: -- A branch with **no push series at all** drops out of the multiplication, so the target whose - worker never ran — the one most worth seeing — is the one this query cannot show. Query the +- A `GitBranch` with **no push series at all** drops out of the multiplication, so the target whose + worker never ran (the one most worth seeing) is the one this query cannot show. Query the mapping against `gitopsreverser_git_queue_depth` for that: ```promql gitopsreverser_git_branch_targets - unless on(provider_namespace, provider_name, branch) gitopsreverser_git_queue_depth + unless on(gitbranch_namespace, gitbranch_name) gitopsreverser_git_queue_depth ``` - `spec.suspend` stops the write and nothing else, so a suspended target keeps its worker, its branch and its series while making no commits by design. Nothing here separates suspended from stuck; `resource_condition` does. +**Which `GitProvider` is failing?** A credential that expired fails every branch its +`GitProvider` writes. Every branch-worker series carries the provider beside the `GitBranch`, which +adds no series (a `GitBranch`'s provider cannot change), so the grouping is one selector: + +```promql +sum by (provider_namespace, provider_name) ( + rate(gitopsreverser_git_pushes_total{outcome="failed"}[15m])) +``` + +**What does the `GitBranch` itself say?** Its conditions are on `resource_condition` under the +generic `resource_namespace` and `resource_name` keys, so rename them to match: + +```promql +label_replace(label_replace( + gitopsreverser_resource_condition{kind="GitBranch", type="Ready", status="False"} == 1, + "gitbranch_namespace", "$1", "resource_namespace", "(.*)"), + "gitbranch_name", "$1", "resource_name", "(.*)") +``` + +A `GitBranch` reporting `BranchConflict` holds no branch and runs no worker, so it has conditions +and no branch-worker series. + **How contended is the branch?** A retry is a replay round *inside* a cycle, not a terminal outcome, which is why it is its own counter rather than a third value on `git_pushes_total`. The ratio is the contention signal: several writers sharing a branch push the remote head forward under @@ -446,10 +469,10 @@ steadily climbing ratio is work being redone, and it is the early warning for th that follows: ```promql -sum by (provider_namespace, provider_name, branch) ( +sum by (gitbranch_namespace, gitbranch_name) ( rate(gitopsreverser_git_push_retries_total[5m])) / -sum by (provider_namespace, provider_name, branch) ( +sum by (gitbranch_namespace, gitbranch_name) ( rate(gitopsreverser_git_pushes_total[5m])) ``` @@ -460,8 +483,8 @@ fixes the Git path; a dropped write is an edit that met a closed render-fidelity ```promql sum by (kind) (rate(gitopsreverser_git_queue_refusals_total[5m])) -sum by (kind, reason) (rate(gitopsreverser_git_commit_failures_total[5m])) -sum by (kind, reason) (rate(gitopsreverser_git_writes_dropped_total[5m])) +sum by (reason) (rate(gitopsreverser_git_commit_failures_total[5m])) +sum by (reason) (rate(gitopsreverser_git_writes_dropped_total[5m])) ``` **How much work is a type's ingestion doing?** A stream is single-threaded, so time it spends @@ -481,14 +504,19 @@ between types rather than as a utilization figure. **Is watch stable, and what are rebuilds costing?** ```promql -sum by (resource, reason) (rate(gitopsreverser_watch_sessions_ended_total[15m])) -histogram_quantile(0.95, sum by (le) (rate(gitopsreverser_watch_replay_duration_seconds_bucket[5m]))) +sum by (source_cluster, resource, reason) (rate(gitopsreverser_watch_sessions_ended_total[15m])) +histogram_quantile(0.95, sum by (le, source_cluster) ( + rate(gitopsreverser_watch_replay_duration_seconds_bucket[5m]))) ``` +`source_cluster` is on these three because the API server that closes, expires and replays a watch +is that cluster's: an `expired` storm is one cluster's watch cache under pressure, and summed across +clusters it reads as everybody's. + A steady trickle of `reason="closed"` is health, not a fault: the API server ends every watch on a randomized timeout and the stream reconnects from its cursor in seconds. Because that reconnect -deliberately moves no condition — a two-second reattach must not flip `StreamsRunning` and make -`kubectl wait --for=condition=Ready` flaky on a healthy cluster — this counter is the **only** place +deliberately moves no condition (a two-second reattach must not flip `StreamsRunning` and make +`kubectl wait --for=condition=Ready` flaky on a healthy cluster), this counter is the **only** place a reconnect is visible, and so the only way to catch a stream that is reconnecting in a hot loop: ```promql @@ -499,23 +527,23 @@ One close per stream per watch timeout is expected; the threshold above is rough magnitude above that. `error` now means a session that ended on a genuine fault, and `expired` remains the one that costs a full rebuild. -`git_commits_total` carries the **`BranchWorker`'s** -`{provider_namespace, provider_name, branch, author_kind, message_source}` identity, not a GitTarget: one worker can -serve several GitTargets sharing a provider+branch, coalescing their writes into one commit batch, so -the worker is the honest attribution unit. `author_kind` is `user`, `serviceaccount`, `committer`, or `unresolved`; -reconcile/resync commits and configured-author mode use `committer`. +`git_commits_total` carries the **`BranchWorker`'s** `GitBranch` and `GitProvider`, plus +`author_kind` and `message_source`, not a GitTarget: one worker serves every GitTarget on its `GitBranch`, coalescing +their writes into one commit batch, so the worker is the honest attribution unit. `author_kind` is +`user`, `serviceaccount`, `committer`, or `unresolved`; reconcile/resync commits and +configured-author mode use `committer`. `message_source` is `live`, `reconcile`, or one of three request values: `commit_request` (a save request's message committed verbatim, no `requestTemplate` configured), `commit_request_framed` (`requestTemplate` rendered), and `commit_request_fallback` (`requestTemplate` failed to render, so the message was committed verbatim instead). -The fallback is the one to alert on. It is a **successful** commit — nothing is refused and no -condition moves — so without this counter a `requestTemplate` that has quietly stopped applying +The fallback is the one to alert on. It is a **successful** commit (nothing is refused and no +condition moves), so without this counter a `requestTemplate` that has quietly stopped applying looks exactly like one that was never configured: ```promql -sum by (provider_namespace, provider_name) ( +sum by (gitbranch_namespace, gitbranch_name) ( rate(gitopsreverser_git_commits_total{message_source="commit_request_fallback"}[15m]) ) > 0 ``` @@ -528,11 +556,11 @@ deliberately not folded into `user` (which would make a lost actor look like a n degrading attribution path would read as an improving one) nor into `committer` (which would hide it among legitimately unattributed reconcile writes). For a live mutation that should be attributable, a non-zero and growing `unresolved` share means the audit-attribution configuration -or delivery path needs investigation. The namespace/name keys are -**prefixed on purpose** — a Prometheus pod scrape with -`honor_labels=false` overwrites a bare `namespace` attribute with the scraping pod's namespace, -so a per-provider `namespace` selector would silently match nothing. The same reasoning applies to -`watch_recovery_total` and `git_queue_depth`. +or delivery path needs investigation. The namespace/name keys are **prefixed on purpose**: a +Prometheus pod scrape with `honor_labels=false` renames a bare `namespace` label to +`exported_namespace` and puts the scraped pod's namespace in its place, so a per-branch `namespace` +selector would silently match nothing. The same reasoning applies to `watch_recovery_total` and +`git_queue_depth`. `message_source` says where each commit's message came from: `commit_request` is text a `CommitRequest` supplied and that was used verbatim, `live` is a live window rendered through the @@ -541,7 +569,7 @@ target's `liveTemplate`, and `reconcile` is a snapshot or resync rendered throug report a source the renderer did not use. `commit_request` counts commits that USED a request-supplied message, **not** CommitRequests. A -request that omits `spec.message` takes the target's `liveTemplate`, so it counts as `live` — a +request that omits `spec.message` takes the target's `liveTemplate`, so it counts as `live`, and a low `commit_request` share does not by itself mean requests are failing. **How much of the history is people naming their own changes?** @@ -551,7 +579,7 @@ sum by (message_source) (rate(gitopsreverser_git_commits_total[15m])) ``` A `commit_request` share that falls to zero after a rollout **may** mean save requests stopped -reaching an open window — but omitted messages, saves that were no-ops, and pushes that never +reaching an open window, but omitted messages, saves that were no-ops, and pushes that never landed all read the same way here, because none of them produce a commit carrying a request message. Confirm against the requests themselves before changing the window: a request that attached reports `Ready=True`, one that produced a pushed commit also reports `Pushed=True` with @@ -583,7 +611,7 @@ metric without bound. `attachTimeout` ran out without a window the request could attach to, but only `window_mismatch` means one was open meanwhile and belonged to somebody else: the author's edits went into that commit, under a **generated** message instead of the sentence they typed. Nothing else in this document -goes red for it. `no_window` is the benign half — nothing was pending to save. +goes red for it. `no_window` is the benign half: nothing was pending to save. The distinction is recorded at expiry from whether a window the request could not attach to was seen during its wait, so a foreign window that closes before `attachTimeout` runs out still counts. Reading @@ -610,7 +638,7 @@ sum by (gittarget_namespace, gittarget_name, close_reason) ( Filter on `timer_source="target"` when tuning a target's own timers. A window a save attached to runs on the save's timers, and a save omits `idleTimeout` by default, so its windows would pull the -picture towards `max_duration`. +picture toward `max_duration`. - `idle_timeout` is the normal ending: the writes went quiet. - `max_duration` means the maximum was reached first. With the target's timers that usually means @@ -622,11 +650,11 @@ picture towards `max_duration`. commits are being split by interleaving rather than by time, and no timer changes that. - `attach_next` is a save with `attach: Next` closing the author's open window before it waits. - `buffer_limit` means the worker's retained-write budget was reached, and the window was flushed - to stay within it. Writes committed but not yet pushed count towards that budget, so this reads + to stay within it. Writes committed but not yet pushed count toward that budget, so this reads as "the branch is holding too much", not as a measurement of pod memory. See [sizing the branch worker queue](#sizing-the-branch-worker-queue-against-git_queue_refusals_total). -- `resync_before_apply` and `atomic_before_apply` close the window so a resync or an atomic write can - apply after it; `shutdown` is the worker stopping. +- `resync_before_apply` closes the window so a resync can apply after it; `shutdown` is the worker + stopping. - `apply_due` closes the window because an apply came due for an applier on the branch: its edits are published at once, ahead of the push cooldown, so the apply includes them. A window a `CommitRequest` holds is never closed this way; the apply waits for the save. Its share follows the @@ -656,10 +684,10 @@ above it: a healthy `5s` window reads between `5s` and `5.5s`, not in the wide ` The buckets then continue up to the `24h` a GitTarget may set. A timer set away from the defaults falls in a wider bucket, and its p95 reads correspondingly coarser. -**Commit rate per provider/branch:** +**Commit rate per GitBranch:** ```promql -sum by (provider_namespace, provider_name, branch) (rate(gitopsreverser_git_commits_total[5m])) +sum by (gitbranch_namespace, gitbranch_name) (rate(gitopsreverser_git_commits_total[5m])) ``` **Are real names landing in Git?** A wall of `author_kind="committer"` means the Git history is not @@ -689,7 +717,7 @@ committing and the folder is relying on steady-state events to catch up: ```promql sum by (gittarget_namespace, gittarget_name) ( - rate(gitopsreverser_git_resync_failures_total[15m])) + rate(gitopsreverser_git_resync_failures_total{reason=~"error|timeout"}[15m])) ``` **How much is each GitTarget watching, and how much of it is actually running?** `sum` is the @@ -700,8 +728,8 @@ sum by (gittarget_namespace, gittarget_name) (gitopsreverser_watch_types) gitopsreverser_watch_types{state="blocked"} > 0 ``` -**Does a target's rules resolve to nothing at all?** The most common misconfiguration there is — a -typo in `resources:`, or a type the default watch policy excludes — produces a target that looks +**Does a target's rules resolve to nothing at all?** The most common misconfiguration there is (a +typo in `resources:`, or a type the default watch policy excludes) produces a target that looks completely healthy on every other panel, because a pipeline with no input has nothing to report: ```promql @@ -723,8 +751,8 @@ resolves **one type** and opens **three watches**. `watch_streams_open` counts t sum by (source_cluster) (gitopsreverser_watch_streams_open) ``` -It counts what the API server itself would count. A session appears when the watch opens — replay -included, since a replaying watch is an open connection — and disappears when it closes, so a +It counts what the API server itself would count. A session appears when the watch opens (replay +included, since a replaying watch is an open connection) and disappears when it closes, so a connection attempt that failed and a stream sitting in reconnect backoff are both absent rather than counted as held. Internal controller and discovery watches are outside it; this is the object-state watch load only. @@ -755,17 +783,17 @@ sum by (group, version, resource) ( ### New-file placement Placement runs **only** for a resource with no document in Git yet; everything already written is -edited in place, forever. So these counters are sparse by nature — a busy target can go a day without -one — and a zero rate is the steady state, not a broken exporter. +edited in place, forever. So these counters are sparse by nature (a busy target can go a day without +one), and a zero rate is the steady state, not a broken exporter. `source` answers "why did it land there?", which is the question a folder cannot answer: | `source` | Means | Needs attention? | | --- | --- | --- | -| `by_type` | a `spec.placement.byType` entry named this exact type | no — this is what you asked for | -| `default` | no `byType` entry named the type, so the catch-all `spec.placement.default` answered | **maybe** — a rule you meant to write may not be matching | -| `kustomize_root` | the folder is governed by exactly one supported kustomization, so the file went beside it and joined its `resources:` list | no — the folder's own structure decided | -| `builtin` | nothing else applied, so the built-in `{namespace}/{group}/{resource}/{name}.yaml` path was used | **maybe** — see below | +| `by_type` | a `spec.placement.byType` entry named this exact type | no: this is what you asked for | +| `default` | no `byType` entry named the type, so the catch-all `spec.placement.default` answered | **maybe**: a rule you meant to write may not be matching | +| `kustomize_root` | the folder is governed by exactly one supported kustomization, so the file went beside it and joined its `resources:` list | no: the folder's own structure decided | +| `builtin` | nothing else applied, so the built-in `{namespace}/{group}/{resource}/{name}.yaml` path was used | **maybe**: see below | **Which types are falling back, and in which target?** Each series is a candidate for one `placement.byType` line. The operator never guesses a hand-authored layout from the folder, so this @@ -778,7 +806,7 @@ sum by (gittarget_namespace, gittarget_name, group, version, resource) ( The built-in path is not an error. For a target whose repository the operator bootstrapped, it is the whole layout and always will be. It is worth acting on when the folder has a convention the operator was not -told about — the file lands somewhere tidy but not where the rest of that type lives. +told about: the file lands somewhere tidy but not where the rest of that type lives. **Is a bundling policy actually bundling?** `disposition="appended"` proves documents are joining an existing file rather than each getting their own. It should only ever appear with `source="by_type"` @@ -803,11 +831,11 @@ sum by (reason, gittarget_namespace, gittarget_name, resource) ( | `plaintext_onto_encrypted` | a template routing a plaintext resource at a file holding SOPS data | | `mixed_sensitivity_new_file` | a bundling `default` catching both a sensitive and a plaintext resource | | `multi_document_target` | the resolved file holds a document the writer cannot account for, so it will not overwrite it | -| `unclassified` | a refusal shape newer than this table — report it | +| `unclassified` | a refusal shape newer than this table; report it | A `{label:key}` template never refuses for a missing label: a resource that does not set it lands at the built-in `_unlabeled` bucket, or at whatever the template's own `{label:key|fallback}` names -— including one directory up, if that fallback is empty — so it is always mirrored somewhere. See +(including one directory up, if that fallback is empty), so it is always mirrored somewhere. See [the label placement docs](layout/new-file-placement-rules.md#labelkey--the-one-metadata-variable) if a lot of resources are landing in `_unlabeled` and you would rather they used a declared fallback or a different variable. @@ -822,7 +850,7 @@ sum by (gittarget_namespace, gittarget_name) ( ``` Placement counters carry `gittarget_*` label keys rather than bare `namespace`/`name` for the -pod-scrape reason described above, and they deliberately carry **no path or resource-name label** — both +pod-scrape reason described above, and they deliberately carry **no path or resource-name label**: both are unbounded, and both are in the log line at the write site. --- @@ -835,15 +863,15 @@ else pushed. See [who applies this folder](configuration.md#who-applies-this-fol | Metric | Type | Labels | Meaning | | --- | --- | --- | --- | -| `apply_requests_total` | counter | `applier_kind`, `reason`, `outcome` | One per attempt to one applier. `applier_kind` is `kustomization` (a Flux `Kustomization`) or `application` (an Argo CD `Application`); `reason` is `ForeignCommit` / `Refusal` / `Repair` / `CatchUp`, for either kind (`CatchUp` adopts an object the API created in the folder, or converges once when the worker starts and once when a `GitTarget` first names the applier claim); `outcome` is `sent` (the applier was asked and the ask landed; it says nothing about how the apply went. What needs no apply, a converge of the commit the applier last applied or a branch the remote does not carry, the branch worker never sends, so it is not counted), `failed` (could not ask, including an applier that applies another repository or branch, an `Application` with automated sync on, or one whose running sync did not end within 5 minutes; the branch worker keeps the reason pending and decides the retry after a flush, so a failure that persists keeps counting at its backoff) or `dropped` (no `GitTarget` may still ask for that reason: each was deleted or recreated, moved to another branch or applier, is no longer admitted by its `KubeProvider`, or withdrew its consent to a `Refusal`) or `stale` (asked nothing because the branch tip moved before the ask landed; the worker decides again at once, at the new tip, and frequent stale attempts mean the branch keeps moving within an attempt's lifetime) | -| `apply_revision_mismatches_total` | counter | `applier_kind` | One per apply operation that succeeded on another revision than the commit GitOps Reverser checked. A `Kustomization` applies whatever its `GitRepository` artifact holds when it reconciles; the artifact is read again just before the resume, so this is somebody fetching the source in between (a person, or the source's own interval). The apply window still withholds a late write over a newer edit; a steady rate means something else keeps fetching the source | -| `claimed_resource_reconciled_total` | counter | `applier_kind`, `claim_namespace`, `claim_name` | One per apply an applier claim did not start: a held `Kustomization` resumed (`flux resume`, or a controller writing its spec without `spec.suspend`), or a held `Application`'s automated sync switched on, by somebody else. From then until the claim takes it back, on its next look within a minute, the applier applies whatever its source holds, unchecked against the branch. The claim records a `ClaimedResourceReconciled` Event for each as well, but the API server folds identical Events into one and expires them, so alert on this counter. A `Kustomization` resumed so also shows the apply as an `Unresolved` operation on the claim | -| `apply_window_collisions_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per write by the applier, while an apply it was asked for runs, to an object the API side edited after the flush before that request: the apply arriving late over a newer edit. `outcome` is `withheld` (proven to be the applier's write, by its field manager alone, with an entry as new as the edit or newer: `kustomize-controller`, or `argocd-controller` / `argocd-application-controller`; so not mirrored: Git keeps the API's value, and a `Repair` of the applier follows, which restores it on the cluster) or `ambiguous` (the applier and another field manager wrote in the same second, so it was mirrored as ever; if it was the apply, the revert is in Git as the applier's commit). An applier entry older than the edit is not counted: it is the API's undo of an edit, mirrored | -| `apply_echoes_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per write by the applier's own field managers alone, judged by the apply window: an apply GitOps Reverser asked for landing, object by object. `outcome` is `not_mirrored` (the applier wrote the object last, at or after the earliest send of the apply windows still open, and nobody edited it in the window: the apply itself, whose content is the commit Git already holds, so it is not mirrored) or `late` (the applier's managers alone wrote the object last with no apply window open: an echo arriving more than 10 seconds after its operation completed, or an apply somebody else started; it is mirrored as an edit). `not_mirrored` rises with every apply and needs no action. A steady `late` rate means applies keep landing outside the windows, and such a write can revert a newer Git-side change to the object: look for a person or a tool applying the folder by hand (`flux reconcile`, a manual sync), or for watch events reaching the worker long after the apply | -| `apply_ownership_conflicts_total` | counter | `provider_namespace`, `provider_name`, `branch` | One per object another applier owns that the branch worker kept out of a coordinated folder: a live write not mirrored, or a snapshot object neither upserted nor swept. The owner is read from Flux's owner labels (`kustomize.toolkit.fluxcd.io/name` and `/namespace`) or Argo CD's tracking annotation (`argocd.argoproj.io/tracking-id`). Any non-zero rate is a misconfiguration: another `Kustomization` or `Application` applies objects into a namespace the folder mirrors, and two appliers applying one object fight each other. The worker logs each object once, naming both appliers; narrow the `WatchRule`, or move the object to one applier | -| `stale_snapshots_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per resync snapshot the branch worker found stale: collected while Git was ahead of the cluster (a commit pushed and not yet applied), collected across an apply, or arriving while Git is ahead. `outcome` is `discarded` (not decided; a fresh snapshot of its `GitTarget` is asked for once Git is no longer ahead, and the `GitTarget`'s live writes are held back by the render-fidelity gate until it lands) or `used` (the `GitTarget`'s snapshots had been held for 5 minutes while an apply kept failing, so this one was decided anyway, and whatever the unapplied commit changed in its objects is reverted, as the API-wins rule says). `discarded` comes in short bursts around applies and needs no action; a discard is not a resync failure and is not counted in `git_resync_failures_total`. Any `used` means an applier is failing for minutes at a time: read the applier claim's `status.operation` and the `ApplyFailed` Events. See [stale snapshots](spec/reconcile-via-watchlist-mark-and-sweep.md#stale-snapshots-are-discarded) | -| `apply_operations_unresolved` | gauge | `provider_namespace`, `provider_name`, `branch` | Apply operations on the branch that have not completed 5 minutes after they were requested. Each is still followed, holds its applier's apply window open, and holds back the applier's next request until it completes, so a value that stays above zero is an applier stuck mid-apply: read its status, and the `ApplyUnresolved` Event on the `GitTarget`s. Read at scrape time | -| `appliers_held` | gauge | `provider_namespace`, `provider_name`, `branch` | Appliers on the branch with a reason pending that the initial-snapshot barrier holds back: no applier is asked for anything until every `GitTarget` naming its claim has had a snapshot of each collection it watches published, and its folder is on the branch. A short hold follows a start, a newly named claim or a watch change. A value that stays above zero is a `GitTarget` that is not mirrored: one whose `WatchRule` selects nothing holds its applier until it watches something, and one whose folder is missing waits for the fresh snapshot the worker asked for (a suspended `GitTarget` never writes it). The worker logs each hold, its reason and its end at Info ("An apply is held back"). Read at scrape time | +| `apply_requests_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `applier_kind`, `claim_namespace`, `claim_name`, `reason`, `outcome` | One per attempt to one applier, labeled by the `GitBranch` whose worker asked and the applier claim holding the applier it asked. `applier_kind` is `kustomization` (a Flux `Kustomization`) or `application` (an Argo CD `Application`); `reason` is `ForeignCommit` / `Refusal` / `Repair` / `CatchUp`, for either kind (`CatchUp` adopts an object the API created in the folder, or converges once when the worker starts and once when a `GitTarget` first names the applier claim); `outcome` is `sent` (the applier was asked and the ask landed; it says nothing about how the apply went. What needs no apply, a converge of the commit the applier last applied or a branch the remote does not carry, the branch worker never sends, so it is not counted), `failed` (could not ask, including an applier that applies another repository or branch, an `Application` with automated sync on, or one whose running sync did not end within 5 minutes; the branch worker keeps the reason pending and decides the retry after a flush, so a failure that persists keeps counting at its backoff) or `dropped` (no `GitTarget` may still ask for that reason: each was deleted or recreated, moved to another branch or applier, is no longer admitted by its `KubeProvider`, or withdrew its consent to a `Refusal`) or `stale` (asked nothing because the branch tip moved before the ask landed; the worker decides again at once, at the new tip, and frequent stale attempts mean the branch keeps moving within an attempt's lifetime) | +| `apply_revision_mismatches_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `applier_kind`, `claim_namespace`, `claim_name` | One per apply operation that succeeded on another revision than the commit GitOps Reverser checked. A `Kustomization` applies whatever its `GitRepository` artifact holds when it reconciles; the artifact is read again just before the resume, so this is somebody fetching the source in between (a person, or the source's own interval). The apply window still withholds a late write over a newer edit; a steady rate means something else keeps fetching the source | +| `apply_claimed_resource_reconciled_total` | counter | `applier_kind`, `claim_namespace`, `claim_name` | One per apply an applier claim did not start: a held `Kustomization` resumed (`flux resume`, or a controller writing its spec without `spec.suspend`), or a held `Application`'s automated sync switched on, by somebody else. From then until the claim takes it back, on its next look within a minute, the applier applies whatever its source holds, unchecked against the branch. The claim records a `ClaimedResourceReconciled` Event for each as well, but the API server folds identical Events into one and expires them, so alert on this counter. A `Kustomization` resumed so also shows the apply as an `Unresolved` operation on the claim | +| `apply_window_collisions_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per write by the applier, while an apply it was asked for runs, to an object the API side edited after the flush before that request: the apply arriving late over a newer edit. `outcome` is `withheld` (proven to be the applier's write, by its field manager alone, with an entry as new as the edit or newer: `kustomize-controller`, or `argocd-controller` / `argocd-application-controller`; so not mirrored: Git keeps the API's value, and a `Repair` of the applier follows, which restores it on the cluster) or `ambiguous` (the applier and another field manager wrote in the same second, so it was mirrored as ever; if it was the apply, the revert is in Git as the applier's commit). An applier entry older than the edit is not counted: it is the API's undo of an edit, mirrored | +| `apply_echoes_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per write by the applier's own field managers alone, judged by the apply window: an apply GitOps Reverser asked for landing, object by object. `outcome` is `not_mirrored` (the applier wrote the object last, at or after the earliest send of the apply windows still open, and nobody edited it in the window: the apply itself, whose content is the commit Git already holds, so it is not mirrored) or `late` (the applier's managers alone wrote the object last with no apply window open: an echo arriving more than 10 seconds after its operation completed, or an apply somebody else started; it is mirrored as an edit). `not_mirrored` rises with every apply and needs no action. A steady `late` rate means applies keep landing outside the windows, and such a write can revert a newer Git-side change to the object: look for a person or a tool applying the folder by hand (`flux reconcile`, a manual sync), or for watch events reaching the worker long after the apply | +| `apply_ownership_conflicts_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | One per object another applier owns that the branch worker kept out of a coordinated folder: a live write not mirrored, or a snapshot object neither upserted nor swept. The owner is read from Flux's owner labels (`kustomize.toolkit.fluxcd.io/name` and `/namespace`) or Argo CD's tracking annotation (`argocd.argoproj.io/tracking-id`). Any non-zero rate is a misconfiguration: another `Kustomization` or `Application` applies objects into a namespace the folder mirrors, and two appliers applying one object fight each other. The worker logs each object once, naming both appliers; narrow the `WatchRule`, or move the object to one applier | +| `apply_stale_snapshots_total` | counter | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name`, `outcome` | One per resync snapshot the branch worker found stale: collected while Git was ahead of the cluster (a commit pushed and not yet applied), collected across an apply, or arriving while Git is ahead. `outcome` is `discarded` (not decided; a fresh snapshot of its `GitTarget` is asked for once Git is no longer ahead, and the `GitTarget`'s live writes are held back by the render-fidelity gate until it lands) or `used` (the `GitTarget`'s snapshots had been held for 5 minutes while an apply kept failing, so this one was decided anyway, and whatever the unapplied commit changed in its objects is reverted, as the API-wins rule says). `discarded` comes in short bursts around applies and needs no action; a discard is not a resync failure and is not counted in `git_resync_failures_total`. Any `used` means an applier is failing for minutes at a time: read the applier claim's `status.operation` and the `ApplyFailed` Events. See [stale snapshots](spec/reconcile-via-watchlist-mark-and-sweep.md#stale-snapshots-are-discarded) | +| `apply_operations_unresolved` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | Apply operations on the branch that have not completed 5 minutes after they were requested. Each is still followed, holds its applier's apply window open, and holds back the applier's next request until it completes, so a value that stays above zero is an applier stuck mid-apply: read its status, and the `ApplyUnresolved` Event on the `GitTarget`s. Read at scrape time | +| `apply_appliers_held` | gauge | `gitbranch_namespace`, `gitbranch_name`, `provider_namespace`, `provider_name` | Appliers on the branch with a reason pending that the initial-snapshot barrier holds back: no applier is asked for anything until every `GitTarget` naming its claim has had a snapshot of each collection it watches published, and its folder is on the branch. A short hold follows a start, a newly named claim or a watch change. A value that stays above zero is a `GitTarget` that is not mirrored: one whose `WatchRule` selects nothing holds its applier until it watches something, and one whose folder is missing waits for the fresh snapshot the worker asked for (a suspended `GitTarget` never writes it). The worker logs each hold, its reason and its end at Info ("An apply is held back"). Read at scrape time | | `receiver_deliveries_total` | counter | `receiver_namespace`, `receiver_name`, `type`, `outcome` | One per verified-or-refused delivery to a `GitReceiver`'s `/git-push/` path. `type` is its `spec.type`; `outcome` is `notified` (handed to at least one branch worker), `unknown_repository` (no `GitProvider`, in any namespace, has the repository the delivery names under any URL it carries), `unknown_branch` (one has, but no branch worker writes that branch for it), `ignored` (a ping, an event other than a push, or a tag or other non-branch ref), `rejected` (a missing or wrong signature or token, or an oversized body) or `invalid` (verified but unparseable, or naming no repository). Every outcome but `rejected` and `invalid` answers `202`. `unknown_repository` and `unknown_branch` are expected on an organization or GitHub App hook, which delivers for every repository and branch. On a repository's own hook, a steady `unknown_repository` rate means the payload names the repository by a URL no provider's `spec.url` matches, such as an SSH alias: the affected `GitBranch` has no `status.lastPushNoticeAt` | A delivery counted `notified` is only a hint: the worker reads the advertisement, and fetches under @@ -853,9 +881,10 @@ followed by no request at all: a push of the worker's own leaves nothing pending already runs the edits it holds. ```promql -# Requests that could not be delivered, by reason. A failure that persists keeps counting, at the -# worker's backoff of up to a few minutes, until the applier is fixed. -sum by (reason) (rate(gitopsreverser_apply_requests_total{outcome="failed"}[15m])) +# Requests that could not be delivered, by applier claim and reason. A failure that persists keeps +# counting, at the worker's backoff of up to a few minutes, until the applier is fixed. +sum by (applier_kind, claim_namespace, claim_name, reason) ( + rate(gitopsreverser_apply_requests_total{outcome="failed"}[15m])) # A receiver a Git host is calling with the wrong secret. sum by (receiver_namespace, receiver_name) ( rate(gitopsreverser_receiver_deliveries_total{outcome="rejected"}[15m])) > 0 @@ -863,34 +892,34 @@ sum by (receiver_namespace, receiver_name) ( # App hook; on one repository's hook it is a URL spelled another way than the provider's. sum by (receiver_namespace, receiver_name) ( increase(gitopsreverser_receiver_deliveries_total{outcome="unknown_repository"}[1h])) > 0 -# Edits landing while an apply runs, by branch. Withheld ones are reverted on the cluster until the +# Edits landing while an apply runs, by GitBranch. Withheld ones are reverted on the cluster until the # Repair that follows; ambiguous ones may have put the applier's revert in Git. -sum by (provider_namespace, provider_name, branch, outcome) ( +sum by (gitbranch_namespace, gitbranch_name, outcome) ( increase(gitopsreverser_apply_window_collisions_total[1h])) > 0 # An applier that somebody else made apply, by claim: each was unchecked against the branch. sum by (applier_kind, claim_namespace, claim_name) ( - increase(gitopsreverser_claimed_resource_reconciled_total[1h])) > 0 + increase(gitopsreverser_apply_claimed_resource_reconciled_total[1h])) > 0 # An applier stuck mid-apply: nothing more is asked of it until its operation completes. -max by (provider_namespace, provider_name, branch) (gitopsreverser_apply_operations_unresolved) > 0 +max by (gitbranch_namespace, gitbranch_name) (gitopsreverser_apply_operations_unresolved) > 0 # Applier writes mirrored as edits because no apply window was open. A steady rate can revert # Git-side changes; something applies the folder outside the windows. -sum by (provider_namespace, provider_name, branch) ( +sum by (gitbranch_namespace, gitbranch_name) ( rate(gitopsreverser_apply_echoes_total{outcome="late"}[1h])) > 0 # Another applier's objects inside a coordinated folder: a misconfiguration, logged per object. -sum by (provider_namespace, provider_name, branch) ( +sum by (gitbranch_namespace, gitbranch_name) ( increase(gitopsreverser_apply_ownership_conflicts_total[1h])) > 0 # Snapshots used although Git was ahead: an applier has been failing for at least 5 minutes. -sum by (provider_namespace, provider_name, branch) ( - increase(gitopsreverser_stale_snapshots_total{outcome="used"}[1h])) > 0 +sum by (gitbranch_namespace, gitbranch_name) ( + increase(gitopsreverser_apply_stale_snapshots_total{outcome="used"}[1h])) > 0 ``` ## Audit attribution (optional) Audit runs when `--author-attribution` is on. The kube-apiserver POSTs audit `EventList` payloads to `/audit-webhook`; the handler applies an intrinsic accept gate and appends the accepted events' -facts to a per-type fact log — **one append per type per request**, not one per event. The watch side +facts to a per-type fact log: **one append per type per request**, not one per event. The watch side follows that log into a bounded, TTL'd in-memory index and joins against it. There is **no body join -and no second source** — watch, not audit, carries the object body — so the only audit metrics are +and no second source** (watch, not audit, carries the object body), so the only audit metrics are the request boundary and the per-event census. The log is Redis Streams by default and an in-process ring with @@ -900,16 +929,16 @@ The log is Redis Streams by default and an in-process ring with | Metric | Type | Labels | | --- | --- | --- | | `audit_eventlist_duration_seconds` | histogram | `outcome` | -| `audit_events_total` | counter | `outcome`, `category`, `group`, `version`, `resource`, `verb` | +| `audit_events_total` | counter | `audit_route`, `outcome`, `category`, `group`, `version`, `resource`, `verb` | | `attribution_resolutions_total` | counter | `tier`, `actor_kind`, `group`, `version`, `resource` | -| `attribution_resolution_wait_seconds` | histogram | `tier`, `event_kind`, `group`, `version`, `resource` | +| `attribution_resolution_wait_seconds` | histogram | `tier`, `event_kind` | | `attribution_facts_total` | counter | `op` | -| `attribution_fact_index_entries` | gauge | — | +| `attribution_fact_index_entries` | gauge | none | | `attribution_fact_index_evictions_total` | counter | `reason` | | `attribution_fact_stream_gaps_total` | counter | `stream` | | `attribution_fact_stream_decode_errors_total` | counter | `transport` | | `attribution_fact_follower_errors_total` | counter | `transport` | -| `attribution_fact_follower_last_success_timestamp_seconds` | gauge | — | +| `attribution_fact_follower_last_success_timestamp_seconds` | gauge | none | | `attribution_collection_without_uidset_total` | counter | `reason` | | `attribution_transport_info` | gauge (always 1) | `transport` | @@ -927,7 +956,7 @@ apiserver posting nothing. `KubeProvider` claims". A named `/audit-webhook/` is accepted as-is, deliberately: a route is a partition name, not a claim about an object, and refusing unknown ones dropped audit batches in flight while a provider was being recreated. The only route-shaped rejection is the bare -`/audit-webhook` endpoint when `--audit-route-annotation-key` is unset. +`/audit-webhook` endpoint when `--author-attribution-audit-route-annotation-key` is unset. ```promql sum by (outcome) (rate(gitopsreverser_audit_eventlist_duration_seconds_count[5m])) @@ -935,22 +964,26 @@ sum by (outcome) (rate(gitopsreverser_audit_eventlist_duration_seconds_count[5m] **Per-event census.** `audit_events_total` increments exactly once per decoded event. `category` is the coarse bucket of `outcome` (carried as its own label so the health invariant is a simple -selector): +selector). `audit_route` is the route the event was posted under, `/audit-webhook/` or the +shared endpoint's annotation, when some `KubeProvider` declares it; `unrecognized` when none does, +and `none` for an event that names no route. A route is producer-chosen text, so only declared ones +become label values. A fresh pod reads every route as `unrecognized` until its `KubeProvider`s have +reconciled, which takes seconds. | `category` | Live `outcome` values | Meaning | | --- | --- | --- | | `stored` | `queued` | Accepted; the event's facts reached the fact log. | -| `dropped` | `nil_event`, `stage`, `read_only_or_unknown_verb`, `failed_request`, `dry_run`, `unchanged_resource_version`, `non_scale_subresource`, `no_attribution_fact` | Correctly rejected at the accept gate, or accepted and unable to name an author — not an error. | -| `error` | `write_error` | The transport rejected the append for THIS event's stream, so its fact did not reach the log. Publication is per stream, so a request that fails partway still counts the events whose own stream appended as `queued`; the whole request is failed so the API server retries it, and the landed facts are simply appended again. The one category that should stay zero. | +| `dropped` | `nil_event`, `stage`, `read_only_or_unknown_verb`, `failed_request`, `dry_run`, `unchanged_resource_version`, `non_scale_subresource`, `missing_cluster_annotation`, `no_attribution_fact` | Correctly rejected at the accept gate, or accepted and unable to name an author; not an error. `missing_cluster_annotation` is an event on the shared, annotation-routed endpoint that names no route; it is never credited to a fallback, so a rising rate is a producer that does not stamp the annotation. | +| `error` | `write_error` | The transport rejected the append for THIS event's stream, so its fact did not reach the log. Publication is per stream, so a request that fails partway still counts the events whose own stream appended as `queued`; the whole request is failed so the API server retries it, and the landed facts are appended again. The one category that should stay zero. | -The full enum lives in [`internal/audit/outcome/outcome.go`](../internal/audit/outcome/outcome.go) -— it is the source of truth. +The full enum lives in [`internal/audit/outcome/outcome.go`](../internal/audit/outcome/outcome.go); +it is the source of truth. **`no_attribution_fact` is the one to read carefully.** The event was accepted and could name no author, so nothing was appended for it and no watch event can ever join it. The usual cause is an aggregated API: the kube-apiserver proxies the request and never decodes the response, so a CREATE's `objectRef` carries no name at all. It is `dropped` rather than `error` because nothing failed, and -this is the only place it can be counted — the event never becomes a fact, so no fact-side counter +this is the only place it can be counted: the event never becomes a fact, so no fact-side counter sees it. A rising share for a type you expect to attribute means commits for that type will be authored unresolved: @@ -964,13 +997,25 @@ sum by (group, resource) (rate(gitopsreverser_audit_events_total{outcome="no_att sum(rate(gitopsreverser_audit_events_total[5m])) ``` -**The health invariant — fact-store errors must be zero:** +**Whose audit stopped?** With several `KubeProvider`s, one cluster's API server can stop posting +while the others carry on, and the total stays positive. Per route: + +```promql +sum by (audit_route) (rate(gitopsreverser_audit_events_total[15m])) == 0 +``` + +A route missing from that result entirely has never posted since the pod started; the +`KubeProvider`'s `AuditFactsReceived` condition says which silence it is in. A steady +`audit_route="unrecognized"` rate is an API server posting under a route no `KubeProvider` carries: +its facts are never read. + +**The health invariant.** Fact-store errors must be zero: ```promql sum(rate(gitopsreverser_audit_events_total{category="error"}[5m])) ``` -**Live audit stream by type — what is actually streaming in.** This is the per-type view of the +**Live audit stream by type: what is actually streaming in.** This is the per-type view of the audit firehose; a type you expect to attribute but never see here means the audit policy is not delivering it: @@ -979,14 +1024,14 @@ sum by (group, version, resource) (rate(gitopsreverser_audit_events_total[5m])) ``` **Have we ever seen audit for this type?** Useful for deciding whether attribution is worth -waiting on for a given type — zero over a long window means audit never delivers it: +waiting on for a given type; zero over a long window means audit never delivers it: ```promql sum by (group, resource) (increase(gitopsreverser_audit_events_total[1h])) > 0 ``` -**What strange or high-volume traffic is the webhook receiving?** Top event shapes by outcome — -surfaces an unexpected flood at a glance: +**What strange or high-volume traffic is the webhook receiving?** Top event shapes by outcome +surface an unexpected flood at a glance: ```promql topk(15, sum by (resource, verb, outcome) (rate(gitopsreverser_audit_events_total[5m]))) @@ -1002,7 +1047,7 @@ topk(10, sum by (resource, verb) ( rate(gitopsreverser_audit_events_total{outcome="non_scale_subresource"}[5m]))) ``` -**Are EventLists failing to decode?** Should be zero — non-zero means a sender is posting +**Are EventLists failing to decode?** Should be zero; non-zero means a sender is posting something that is not an `audit.k8s.io/v1 EventList`: ```promql @@ -1026,7 +1071,7 @@ sum(rate(gitopsreverser_attribution_resolutions_total[5m])) ``` Coverage is `tier!="absent"` and nothing narrower. `tier=~"exact.*"` reads the collection and name -tiers as misses, which they are not — they named an actor. +tiers as misses, which they are not: they named an actor. The two labels answer two different questions. **`tier`** names which evidence produced the author, and it is ordered, strongest first. **`actor_kind`** names who that evidence named, in the same @@ -1049,15 +1094,15 @@ equally hold a write, and a value that could mean either must not claim a verb. | `exact` | Exact UID+resourceVersion match: this actor produced this exact version. For a removal of an object that was not terminating, it is the write that took the object out of view: a label exit. | | `deletecollection_body_uid` | A removal whose UID was in the set the API server said a `deletecollection` deleted. No over-attribution risk: either the object was in that set or it was not. It outranks `latest`, because `latest` names whoever last *wrote* an object while a removal asks who *deleted* it. | | `latest` | The UID-latest tier: the object's own last fact, keyed by UID alone. A removal accepts it only when that fact is a delete, which `delete_sticky` normally answers first, so it appears only when the sticky pointer was evicted under the index caps. A *write* here never answers a removal. | -| `name` | A match on `(namespace, name)` for a fact carrying neither a UID nor a resourceVersion — the usual shape of an aggregated API's audit event, and of a delete the API server answered with a `Status`. | -| `deletecollection_scope` | A removal matched to a `deletecollection` by scope alone — same type and namespace, the request's selector accepting the object's labels, within the collection window. The weakest evidence the join has, and the only one that can name the wrong actor, which is why it is reached only when every more specific tier missed. | +| `name` | A match on `(namespace, name)` for a fact carrying neither a UID nor a resourceVersion: the usual shape of an aggregated API's audit event, and of a delete the API server answered with a `Status`. | +| `deletecollection_scope` | A removal matched to a `deletecollection` by scope alone (same type and namespace, the request's selector accepting the object's labels, within the collection window). The weakest evidence the join has, and the only one that can name the wrong actor, which is why it is reached only when every more specific tier missed. | | `resource_version` | The RV-only escape hatch: a fact that carried a resourceVersion and no UID, matched on that version alone. | | `absent` | No usable fact matched before the grace window elapsed. The resulting live commit is authored as `unknown (attribution unresolved)`. | | `actor_kind` | Meaning | | --- | --- | | `user` | A human (or any non-service-account subject). | -| `serviceaccount` | A named service account — a controller, an operator, a CI identity. | +| `serviceaccount` | A named service account: a controller, an operator, a CI identity. | | `none` | Nobody was named, which in practice means nothing matched. | `actor_kind="none"` and `tier="absent"` go together, and that is an invariant rather than a @@ -1112,7 +1157,7 @@ pressure, and eviction falls on the type holding the most). This counter is also the removal pointer's only horizon. Every other entry in the index expires on the fact TTL; a removal pointer is bounded by these caps instead, so a cluster that deletes rarely -keeps its removals for a very long time at no cost, and a busy one keeps the most recent — which are +keeps its removals for a very long time at no cost, and a busy one keeps the most recent, which are the ones a replay is likeliest to need. A sustained eviction rate on a type that deletes heavily is the signal that its pointers are being reclaimed before a replay can use them. @@ -1126,15 +1171,15 @@ sum by (stream) (rate(gitopsreverser_attribution_fact_stream_gaps_total[5m])) ``` **Is a follower silently skipping facts?** An entry the follower refuses is skipped and its -position passed — which is right, since it can never decode and stalling on it would cost every -later fact on that stream — so the facts it carried are gone. This is the loss path with **no other +position passed (which is right, since it can never decode and stalling on it would cost every +later fact on that stream), so the facts it carried are gone. This is the loss path with **no other symptom**: unlike a trim gap it is not detectable after the fact, and unlike a publish failure the API server does not retry it. Two things are refused: a payload that is not valid JSON, and one that is JSON but breaks the fact -contract by naming nobody (`author` must be present and non-empty — a fact exists to name somebody). +contract by naming nobody (`author` must be present and non-empty: a fact exists to name somebody). Either way the log line names the stream and the entry. Any non-zero rate means something is writing -entries this operator would not write — a version skew, another producer, or a hand-edited stream: +entries this operator would not write: a version skew, another producer, or a hand-edited stream: ```promql sum by (transport) (rate(gitopsreverser_attribution_fact_stream_decode_errors_total[5m])) @@ -1156,8 +1201,8 @@ sum by (transport) (rate(gitopsreverser_attribution_fact_follower_errors_total[5 ``` **An absent gauge is the worst case, not a healthy one.** The timestamp is not published until the -follower's first successful read, so a follower that has been wedged since startup — a transport -unreachable at boot — has no series at all, and `time() - ` therefore returns nothing rather +follower's first successful read, so a follower that has been wedged since startup (a transport +unreachable at boot) has no series at all, and `time() - ` therefore returns nothing rather than a large number. An alert must cover that arm explicitly, which is what `attribution_transport_info` is for: it is published when the follower starts, so a transport running without a last-success timestamp is exactly the never-succeeded case. Give it a `for: 10m` so an ordinary restart's gap does @@ -1181,7 +1226,7 @@ gitopsreverser_attribution_transport_info **How often does a collection delete fall back to scope matching?** A `deletecollection` fact carries the UIDs the API server named, when it sent them, and joins by membership. When it cannot, -the join falls back to `deletecollection_scope`, which is correct but weaker — so the fallback is counted +the join falls back to `deletecollection_scope`, which is correct but weaker, so the fallback is counted rather than inferred: ```promql @@ -1189,9 +1234,9 @@ sum by (reason) (rate(gitopsreverser_attribution_collection_without_uidset_total ``` `reason` is `uid_cap` (the set was larger than `--author-attribution-collection-uid-cap`) or -`no_uids` (the API server sent a body with no usable UIDs). A body-less response — a truncated, -aggregated, or metadata-only `deletecollection` — produces no fact-level degradation event at all, -because there was never a set to drop; it simply resolves through the scope tier. A production +`no_uids` (the API server sent a body with no usable UIDs). A body-less response (a truncated, +aggregated, or metadata-only `deletecollection`) produces no fact-level degradation event at all, +because there was never a set to drop; it resolves through the scope tier. A production cluster running `--audit-webhook-truncate-enabled` is the one most likely to be in that case. --- @@ -1199,7 +1244,7 @@ cluster running `--audit-webhook-truncate-enabled` is the one most likely to be ## API resource catalog The API resource catalog is GitOps Reverser's single trusted in-memory view of the cluster's -served API surface — every `WatchRule` and `ClusterWatchRule` is resolved against it. The watch +served API surface: every `WatchRule` and `ClusterWatchRule` is resolved against it. The watch manager refreshes it from Kubernetes discovery on its 30 s reconcile ticker, on every CRD/APIService change, and on every rule change. @@ -1210,7 +1255,7 @@ CRD/APIService change, and on every rule change. | `api_catalog_refresh_total` | counter | `source_cluster`, `outcome` (`changed`/`unchanged`/`error`) | | `api_catalog_refresh_duration_seconds` | histogram | `source_cluster` | -`excluded` resources are the default-watch-policy set (pods, events, leases, jobs, …) — served +`excluded` resources are the default-watch-policy set (pods, events, leases, jobs, …): served by the cluster but deliberately never watched. `degraded` group/versions are ones discovery reported as failed, usually a broken aggregated APIService. @@ -1234,7 +1279,7 @@ sum by (outcome) (rate(gitopsreverser_api_catalog_refresh_total[15m])) gitopsreverser_api_catalog_group_versions{state="degraded"} > 0 ``` -**Discovery latency** — catches a slow or non-aggregated apiserver. Normal is single-digit +**Discovery latency** catches a slow or non-aggregated apiserver. Normal is single-digit milliseconds; the call is two cached GETs on Kubernetes ≥ 1.27: ```promql @@ -1246,7 +1291,7 @@ histogram_quantile(0.95, ## Watch plane owner -One loop owns the watch plane: controllers post an applier claim naming a GitTarget and return, and the +One loop owns the watch plane: controllers post a request naming a GitTarget and return, and the owner runs one plan pass per target once that target has been quiet for a 2 s rolling silence window (hard cap ~10 s). Design: [watch-manager-ownership.md](design/watch-manager-ownership.md). @@ -1256,15 +1301,15 @@ five make it legible. | Metric | Type | Labels | | --- | --- | --- | -| `watch_plan_dirty_targets` | gauge | — | -| `watch_plan_oldest_dirty_since_timestamp_seconds` | gauge | — | +| `watch_plan_dirty_targets` | gauge | none | +| `watch_plan_oldest_dirty_since_timestamp_seconds` | gauge | none | | `watch_plan_passes_total` | counter | `outcome` (`completed`/`failed`/`timed_out`), `gittarget_namespace`, `gittarget_name` | -| `watch_plan_pass_duration_seconds` | histogram | — | -| `watch_plan_triggers_total` | counter | `reason` (`declare`/`rule_change`/`shared_refresh`/`periodic`), `coalesced` | +| `watch_plan_pass_duration_seconds` | histogram | none | +| `watch_plan_triggers_total` | counter | `reason` (`declare`/`rule_change`/`shared_refresh`/`periodic`/`worker_replaced`), `coalesced` | **Is anything stuck?** This is the one to alert on. The oldest dirty target's age climbs and stays -up exactly when a target cannot be planned — a source cluster that stopped answering, a catalog -that never becomes observable — whether the queue is deep or holds a single wedged target. A +up exactly when a target cannot be planned (a source cluster that stopped answering, a catalog +that never becomes observable), whether the queue is deep or holds a single wedged target. A healthy install sits under the settle window plus a pass, so seconds: ```promql @@ -1292,8 +1337,8 @@ sum by (gittarget_namespace, gittarget_name) ( ``` **Is the silence window earning its keep?** One `kubectl apply` of a GitTarget and four WatchRules -should show four coalesced applier claims and one pass. A coalescing ratio near zero under a busy config -plane means edits are arriving further apart than the window, which is not a fault — it is the +should show four coalesced triggers and one pass. A coalescing ratio near zero under a busy config +plane means edits are arriving further apart than the window, which is not a fault: it is the window doing nothing: ```promql @@ -1303,14 +1348,16 @@ sum(rate(gitopsreverser_watch_plan_triggers_total{coalesced="true"}[15m])) **Which source is noisy?** `periodic` is the 30 s floor and is expected to dominate a quiet cluster. A high `shared_refresh` rate means the API surface is flapping; a high `rule_change` rate -means something is rewriting rules: +means something is rewriting rules. `worker_replaced` follows a `GitProvider` recreated for another +repository: the new branch worker has no clone, so every `GitTarget` on the branch re-establishes +its folder: ```promql sum by (reason) (rate(gitopsreverser_watch_plan_triggers_total[15m])) ``` **How long does a pass take?** The input to choosing the per-target deadline. A pass is in-memory -replanning — discovery is a shared job done once per cluster, not per target — so normal is +replanning (discovery is a shared job done once per cluster, not per target), so normal is sub-millisecond, and anything near a second means a target is doing I/O it should not be: ```promql @@ -1326,9 +1373,9 @@ Secrets are never committed in plaintext; these metrics confirm the encryption p | Metric | Type | Notes | | --- | --- | --- | -| `secret_encryptions_total` | counter | One per encryption decision, labeled `outcome`: `encrypted`, `failed` (the write is rejected), or `cached` (unchanged content reused). | +| `secret_encryptions_total` | counter | One per encryption decision, labeled by the `GitTarget` (`gittarget_namespace`, `gittarget_name`) and `outcome`: `encrypted`, `failed` (the write is rejected), or `cached` (unchanged content reused). | -**Encryption failure rate** — should be zero; non-zero means Secret writes are being rejected: +**Encryption failure rate** should be zero; non-zero means Secret writes are being rejected: ```promql rate(gitopsreverser_secret_encryptions_total{outcome="failed"}[5m]) @@ -1347,28 +1394,38 @@ sum by (outcome) (rate(gitopsreverser_secret_encryptions_total[5m])) | Condition | Meaning | | --- | --- | -| `rate(gitopsreverser_audit_events_total{category="error"}[10m]) > 0` | Attribution fact-store writes are failing — check Redis. | +| `rate(gitopsreverser_audit_events_total{category="error"}[10m]) > 0` | Attribution fact-store writes are failing; check Redis. | +| `sum by (audit_route) (rate(gitopsreverser_audit_events_total{audit_route!~"unrecognized\|none"}[15m])) == 0`, `for: 15m` | One cluster's API server stopped posting audit while it still did earlier in the pod's life; its commits are authored unresolved. A cluster whose audit policy forwards nothing for 15 minutes reads the same way. | | `rate(gitopsreverser_audit_eventlist_duration_seconds_count{outcome="decode_error"}[10m]) > 0` | A sender is posting non-EventList payloads to `/audit-webhook`. | | `rate(gitopsreverser_audit_eventlist_duration_seconds_count{outcome=~"bad_method\|bad_path\|bare_endpoint_disabled"}[15m]) > 0` | An apiserver is posting audit to a path this operator does not serve. | | `rate(gitopsreverser_attribution_fact_stream_decode_errors_total[10m]) > 0` | A schema or version mismatch on the fact stream; facts are being skipped and lost. | -| `(time() - …_fact_follower_last_success_timestamp_seconds > 600) or (…_transport_info == 1 unless on() …_fact_follower_last_success_timestamp_seconds)`, `for: 10m` | The fact follower is wedged; attribution is degrading to committer-authored cluster-wide. Both arms are needed — see below. | -| `rate(gitopsreverser_git_resync_failures_total[15m]) > 0` sustained | Background resyncs are not committing; the folder relies on steady-state events to catch up. | +| `(time() - …_fact_follower_last_success_timestamp_seconds > 600) or (…_transport_info == 1 unless on() …_fact_follower_last_success_timestamp_seconds)`, `for: 10m` | The fact follower is wedged; attribution is degrading to committer-authored cluster-wide. Both arms are needed; see below. | +| `rate(gitopsreverser_git_resync_failures_total{reason=~"error\|timeout"}[15m]) > 0` sustained | Background resyncs are not committing; the folder relies on steady-state events to catch up. | | `increase(gitopsreverser_apply_ownership_conflicts_total[1h]) > 0` | Another `Kustomization` or `Application` applies objects into a coordinated folder; they are kept out of Git, and the log names both appliers. | -| `increase(gitopsreverser_claimed_resource_reconciled_total[1h]) > 0` | Somebody else made a held applier apply, unchecked against the branch; the labels name the applier claim. A steady rate on one claim is a controller that rewrites the `Kustomization` without `spec.suspend`. | -| `increase(gitopsreverser_stale_snapshots_total{outcome="used"}[1h]) > 0` | An applier has been failing for at least 5 minutes, so a resync decided a snapshot taken while Git was ahead and reverted part of the unapplied commit. | +| `increase(gitopsreverser_apply_claimed_resource_reconciled_total[1h]) > 0` | Somebody else made a held applier apply, unchecked against the branch; the labels name the applier claim. A steady rate on one claim is a controller that rewrites the `Kustomization` without `spec.suspend`. | +| `increase(gitopsreverser_apply_stale_snapshots_total{outcome="used"}[1h]) > 0` | An applier has been failing for at least 5 minutes, so a resync decided a snapshot taken while Git was ahead and reverted part of the unapplied commit. | | `gitopsreverser_api_catalog_group_versions{state="degraded"} > 0` | Part of the API surface is hidden behind a broken APIService. | -| `time() - gitopsreverser_watch_plan_oldest_dirty_since_timestamp_seconds > 120`, `for: 5m` | A GitTarget cannot be planned — its source cluster or catalog is unreachable — and its mirror is not converging. The `time() -` is not optional: the gauge is a Unix timestamp, so a bare `> 120` is true of every instant since 1970 and would fire on ordinary planning. | +| `time() - gitopsreverser_watch_plan_oldest_dirty_since_timestamp_seconds > 120`, `for: 5m` | A GitTarget cannot be planned (its source cluster or catalog is unreachable), and its mirror is not converging. The `time() -` is not optional: the gauge is a Unix timestamp, so a bare `> 120` is true of every instant since 1970 and would fire on ordinary planning. | | `rate(gitopsreverser_watch_plan_passes_total{outcome="timed_out"}[15m]) > 0` sustained | Plan passes are hitting their per-target deadline; the target stays dirty and installs nothing. | | `rate(gitopsreverser_secret_encryptions_total{outcome="failed"}[10m]) > 0` | Secret writes are being rejected by the encryption path. | -| `rate(gitopsreverser_git_queue_refusals_total[5m]) > 0 unless on(provider_namespace, provider_name, branch) gitopsreverser_git_intake_paused == 1`, `for: 10m` | A healthy branch keeps refusing work: its queue is too small for its bursts. Producers offer the work again, so this is lag; only a watch cursor that expires first loses commits, and under `OnEvent` deletes. During an outage the intake-paused alert below covers it. | +| `rate(gitopsreverser_git_queue_refusals_total[5m]) > 0 unless on(gitbranch_namespace, gitbranch_name) gitopsreverser_git_intake_paused == 1`, `for: 10m` | A healthy branch keeps refusing work: its queue is too small for its bursts. Producers offer the work again, so this is lag; only a watch cursor that expires first loses commits, and under `OnEvent` deletes. During an outage the intake-paused alert below covers it. | | `gitopsreverser_git_queue_depth` rising and not draining | A branch worker is backing up against a stalled remote. | | `time() - gitopsreverser_git_oldest_retained_write_timestamp_seconds > 900`, `for: 5m` | A branch has not published for 15 minutes: its writes, and any save riding them, are held. The `time() -` is not optional, as for the plan-dirty gauge above. | | `gitopsreverser_git_intake_paused == 1`, `for: 10m` | An outage filled a branch's budget: new changes on its `GitTarget`s are refused until a push lands. | +| `sum by (gitbranch_namespace, gitbranch_name, reason) (increase(gitopsreverser_git_commit_failures_total[15m])) > 0` | Loss: a window died between routing and pushing, and its edits are missing from Git until the next resync. `reason="refused"` will not clear until someone fixes the Git path. | +| `sum by (gitbranch_namespace, gitbranch_name, reason) (increase(gitopsreverser_git_writes_dropped_total[15m])) > 0` | Loss: writes met a closed render-vs-live gate and were dropped, and nothing replays them. `render_fidelity_diverged` goes with `RenderMatchesLive=False` on a `GitTarget`; `render_fidelity_pending` is a hold that gave up. | +| `sum by (gittarget_namespace, gittarget_name, reason) (increase(gitopsreverser_placement_refusals_total[1h])) > 0` | Loss: new resources the writer declined to place are absent from the mirror until the placement policy is fixed. | +| `sum by (gittarget_namespace, gittarget_name) (increase(gitopsreverser_placement_kustomization_entries_total{outcome="failed"}[1h])) > 0` | A new file is in Git without its `resources:` entry, so kustomize never builds it. | +| `max by (gitbranch_namespace, gitbranch_name) (gitopsreverser_apply_operations_unresolved) > 0`, `for: 15m` | An applier is stuck mid-apply, and nothing more is asked of it until the operation completes. | +| `max by (gitbranch_namespace, gitbranch_name) (gitopsreverser_apply_appliers_held) > 0`, `for: 15m` | An applier is held behind the initial-snapshot barrier: a `GitTarget` naming its claim watches nothing, or its folder is missing. | +| `sum by (applier_kind, claim_namespace, claim_name, reason) (rate(gitopsreverser_apply_requests_total{outcome="failed"}[15m])) > 0`, `for: 30m` | Apply requests keep failing for the named applier claim; the worker retries at its backoff, and the cluster runs older content than the branch meanwhile. | +| `sum by (receiver_namespace, receiver_name) (increase(gitopsreverser_receiver_deliveries_total{outcome="rejected"}[15m])) > 0` | A Git host is calling a `GitReceiver` with the wrong secret, so foreign pushes are found only by the periodic refresh. | +| `sum by (gitbranch_namespace, gitbranch_name) (rate(gitopsreverser_git_commits_total{message_source="commit_request_fallback"}[15m])) > 0` | A `requestTemplate` stopped rendering: saves still commit, with the message verbatim. | | `gitopsreverser_resource_condition{type="Ready", status="False"} == 1`, `for: 15m` | A declared object has not been accepted. The `== 1` is not optional: each condition publishes one series per status, so the selector alone matches the two zeroes too. | | `gitopsreverser_resource_condition{type="Stalled", status="True"} == 1` | Permanently wedged: `Stalled` is the kstatus "nothing will retry" signal, so this needs a human and will not clear on its own. `reason` names the gate. | | any `gitopsreverser_*` series with `otel_metric_overflow="true"` | A metric family exceeded its per-instrument cardinality cap, so the labels identifying those series are gone and every query over that family is under-reporting. Raise the cap or cut a label. | | `sum by (group, resource) (rate(gitopsreverser_attribution_resolutions_total{tier="absent"}[10m])) > 0`, `for: 15m` | Events of this type keep waiting out the whole `--author-attribution-grace` and shipping `unknown (attribution unresolved)`. Each one held its watch stream for the full grace, so every later event of the type, including a save's edit, reached its branch worker that much later. Check the audit policy for the type and the `KubeProvider`'s audit route before anything else. An occasional absence is ordinary; a sustained rate is a configuration problem or a fact that can never arrive. | -| `sum by (gittarget_namespace, gittarget_name) (gitopsreverser_watch_types) == 0`, `for: 10m` | A GitTarget's rules resolve to no watchable type — a typo in `resources:`, or a type the default watch policy excludes. It mirrors nothing while looking healthy everywhere else. The 10 minutes let a freshly declared target settle. | +| `sum by (gittarget_namespace, gittarget_name) (gitopsreverser_watch_types) == 0`, `for: 10m` | A GitTarget's rules resolve to no watchable type: a typo in `resources:`, or a type the default watch policy excludes. It mirrors nothing while looking healthy everywhere else. The 10 minutes let a freshly declared target settle. | --- @@ -1394,6 +1451,6 @@ Listed so a missing panel is never read as a healthy zero. The plan for them is The bar for a row here: a reader who has never seen the metric learns what it measures, one query that answers a real operator question, and what a bad value looks like. A metric without an -interpretation is noise. The rest of the policy — a recording site before an instrument, and the -doc row in the same change — is +interpretation is noise. The rest of the policy (a recording site before an instrument, and the +doc row in the same change) is [metrics-observability-plan.md](design/metrics-observability-plan.md) §3. diff --git a/docs/review/20261026-metrics-and-labels.md b/docs/review/20261026-metrics-and-labels.md new file mode 100644 index 000000000..8b17780be --- /dev/null +++ b/docs/review/20261026-metrics-and-labels.md @@ -0,0 +1,95 @@ +# Review: the metrics and their labels, before the breaking release + +Every instrument GitOps Reverser exports was read at its recording site. For each one, this review +asks whether it answers a question an operator has, and whether its labels let that operator find +the object to fix. This release breaks the API anyway, so a label change costs the least now. Every +finding below was acted on in the same branch. + +## Scope + +Reviewed on `docs-metrics-architecture-inventory` (PR #450) from `918fa804`: the 65 instruments +registered in [`internal/telemetry/exporter.go`](../../internal/telemetry/exporter.go) (38 +counters, 8 histograms, 5 synchronous gauges, 14 observable gauges), their recording sites, and +[`interpreting-metrics.md`](../interpreting-metrics.md). The cardinality model is +[§7.1 of the plan](../design/metrics-observability-plan.md#71-cardinality-budget). + +## Verdict + +The surface is in good shape. Each stage of the pipeline counts once with a bounded `outcome`, +the loss paths are separate counters, the saturation gauges are read at scrape time, and every +instrument in the guide has an interpretation and a query. None of them is noise: each answers a +question that no condition, Event or log line answers as well. + +The labels had one structural flaw. The branch worker's 21 families were labeled by +`{provider_namespace, provider_name, branch}`, a label set from before `GitBranch` existed. It did +not name the object an operator acts on, so a failing push could not be matched to the +`GitBranch` that reports it. The smaller findings follow. + +## The branch worker's identity + +Every branch-worker series now carries `{gitbranch_namespace, gitbranch_name, provider_namespace, +provider_name}`, set in one place (`BranchWorker.gitBranchAttrs`). The `GitBranch` is the object to +act on; the `GitProvider` groups the branches one credential writes, which is how an expired +credential shows. Neither adds a series: a worker serves one `GitBranch`, and a `GitBranch`'s +provider is immutable. `branch` is dropped because the `GitBranch` already names it. + +A first version of this change dropped the provider too and published it on a separate +`git_branch_info` series for a join. That was undone: the join cost every "by provider" query a +`group_left`, and a `group_left` errors when two pods export the info series at once, as during a +rollout. Labels that add no series belong on the series. + +`git_branch_targets` maps a `GitTarget` to its `spec.gitBranchRef`, so it names the target even +when the `GitBranch` or its provider does not resolve. Before, it published empty provider labels +in that case. + +## Findings + +| # | Finding | What was done | +| --- | --- | --- | +| R1 | `audit_events_total` carried no cluster. With several `KubeProvider`s posting audit, "whose audit stopped" could not be asked. | It gains `audit_route`: the route an event was posted under, when a live `KubeProvider` declares it (the reconcile publishes each provider's route into `attribution.RouteHealth`); `unrecognized` for any other route; `none` for a shared-endpoint event naming none. The raw route is producer-chosen, so it is never a label unchecked. The request histogram stays route-free: a histogram pays its bucket count per label. | +| R2 | `git_fetches_total{reason="bootstrap"}` could never emit. Its caller, `BranchWorker.EnsurePathBootstrapped`, had no production caller, and neither did `ensureRepositoryInitialized`: a worker's first contact is the publication cycle's fetch. | The dead entry point, its five helpers and the reason are deleted. `ensureRepositoryInitialized` moved to a test fixture. The template staging it wrapped is live, and its tests now call `ensureBootstrapTemplateInPath` directly. Ledger rows 1 and 2 keep their numbers, which docs cite, and measure the same two requests. | +| R3 | Suspected: `git_resync_failures_total` overlaps `git_commit_failures_total`. Traced: it does not, because a resync's commit failure never reaches `recordCommitFailure`. It did overlap two others: a resync refused at intake is also `git_queue_refusals_total{kind="resync"}`, and a resync deferred while the remote is unreachable stays pending, yet was counted as failed. | `git_resync_failures_total` gains `reason`: `error` and `timeout` lose a snapshot; `refused` and `deferred` lose nothing. A deferred reply wraps the new `git.ErrResyncDeferred`. The alert selects `reason=~"error\|timeout"`. | +| R4 | `attribution_resolution_wait_seconds` carried `tier`, `event_kind` and the type triple: as a histogram, the largest family in §7.1 (7,680 series in the model install). No query used the type on it. | The type is gone from the histogram; `attribution_resolutions_total` keeps it. | +| R5 | `secret_encryptions_total{outcome="failed"}` named no `GitTarget`. | It gains `gittarget_namespace` and `gittarget_name`, from the write batch's own target, which every path sets. | +| R6 | `InitOTLPExporter` set up a Prometheus exporter and printed to stdout. Nothing is pushed over OTLP. | Renamed `InitPrometheusExporter`; the prints are gone, and the no-op shutdown says why it is one. | +| R7 | `appliers_held`, `stale_snapshots_total` and `claimed_resource_reconciled_total` had no `apply_` prefix. | Renamed `apply_appliers_held`, `apply_stale_snapshots_total` and `apply_claimed_resource_reconciled_total`. | +| R8 | New, found tracing R3: nothing in production sent an atomic write request (`CommitModeAtomic`), so `kind="atomic"` on `git_commit_failures_total` and `git_writes_dropped_total`, and `close_reason="atomic_before_apply"`, never emitted. The commit-failure doc also called `atomic` "a snapshot/resync request", which it was not. | The atomic write path is deleted: `handleAtomicRequest`, `buildAtomicPendingWrite`, `PendingWriteAtomic`, `CommitMode`, and every `WriteRequest` field but `Events`. With one value left, `kind` comes off both counters. Its enqueue logs now name the event's GitTarget; they printed an always-empty request field. Tests of shared behavior were moved onto live writes. | + +Other changes in the same pass: + +- `deadcode -test -tags e2e ./...` found four functions nothing calls, in tests or any binary: + `git.AuthFromSecretData`, `git.GetCommitSigner`, `giteaclient.Client.RegisterUserKeyAsUser` and + `test/utils.UncommentCode`. They are deleted, and the sweep now reports nothing. + +- `apply_requests_total` and `apply_revision_mismatches_total` had no identity at all. They carry + the `GitBranch` and provider that asked and the claim that was asked. +- `watch_sessions_ended_total`, `watch_replay_duration_seconds` and `watch_event_handling_seconds` + gain `source_cluster`: an `expired` storm is one API server's watch cache under pressure. +- The rule "object names are labels in one family only" was already false. It now says what §7.1 + says: configuration objects may be labels, mirrored resources never are. + +## Family by family + +**Configuration state** (`resource_condition`). The best-designed family: status as a label, +`Unknown` synthesized, deleted objects stop publishing. It keeps its generic `resource_namespace` +and `resource_name` keys, the Flux shape; matching a `GitBranch`'s conditions to its worker series +takes a `label_replace`, which the guide shows. + +**Watch ingest.** Labeled by the `GitTarget` and the type, which is what "which tenant stopped +receiving events" needs; the per-type stream instruments now carry the cluster. + +**Write boundary and windows.** Correctly per `GitTarget`: a document, a placement, a window and a +save each belong to one target. + +**Branch worker and apply.** Useful and well split: pushes, retries, fetches, refusals, losses and +the retained backlog each answer a different question. + +**Audit attribution.** The fact-loss trio stays three counters on purpose: their units differ. + +**Catalog, watch plane, Secret encryption, receiver.** Right as they are, with R5 applied. + +## Consequence to plan for + +A dashboard or alert that selects a branch-worker series by `branch`, or names one of the three +renamed apply series, matches nothing after this release. +[`UPGRADING.md`](../UPGRADING.md) lists every series that changed. diff --git a/docs/spec/reconcile-via-watchlist-mark-and-sweep.md b/docs/spec/reconcile-via-watchlist-mark-and-sweep.md index 1d4b85147..e7d3dc4bb 100644 --- a/docs/spec/reconcile-via-watchlist-mark-and-sweep.md +++ b/docs/spec/reconcile-via-watchlist-mark-and-sweep.md @@ -367,7 +367,7 @@ Three rules keep this from holding a resync forever: Only the worker's own appliers count: other `GitProvider`s writing the same repository and branch are not consulted. Both outcomes are counted in -`stale_snapshots_total` (`discarded`, and `used` for the 5-minute bound). Design: +`apply_stale_snapshots_total` (`discarded`, and `used` for the 5-minute bound). Design: [applier echoes, F3](../design/applier-echoes.md#f3-a-resync-never-decides-from-a-snapshot-collected-while-git-was-ahead-or-across-an-apply). ## Lazy Materialization Synergy diff --git a/internal/applier/dispatcher.go b/internal/applier/dispatcher.go index c780c2358..c5173faa9 100644 --- a/internal/applier/dispatcher.go +++ b/internal/applier/dispatcher.go @@ -662,7 +662,7 @@ func (d *Dispatcher) completed( result.Message = progress.Message if telemetry.ApplyRevisionMismatchesTotal != nil { telemetry.ApplyRevisionMismatchesTotal.Add(context.WithoutCancel(ctx), 1, - metric.WithAttributes(attribute.String("applier_kind", applierKindOf(req)))) + metric.WithAttributes(applyAttrs(req)...)) } } else { op.log.V(1).Info("the applier finished the requested apply", "after", took.String()) @@ -872,11 +872,26 @@ func (d *Dispatcher) count(ctx context.Context, req git.ApplyRequest, outcome st if telemetry.ApplyRequestsTotal == nil { return } - telemetry.ApplyRequestsTotal.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes( - attribute.String("applier_kind", applierKindOf(req)), + telemetry.ApplyRequestsTotal.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(applyAttrs(req, attribute.String("reason", string(req.Reason)), attribute.String("outcome", outcome), - )) + )...)) +} + +// applyAttrs is the identity an apply attempt is counted under: the GitBranch whose worker asked and +// its GitProvider, as every branch-worker series names them, and the applier claim it asked, as +// apply_claimed_resource_reconciled_total names it. Neither invents an owner: one worker decides the +// attempt and one claim holds the applier, so the pair adds no series beyond the claims. +func applyAttrs(req git.ApplyRequest, extra ...attribute.KeyValue) []attribute.KeyValue { + return append([]attribute.KeyValue{ + attribute.String("gitbranch_namespace", req.GitBranch.Namespace), + attribute.String("gitbranch_name", req.GitBranch.Name), + attribute.String("provider_namespace", req.Provider.Namespace), + attribute.String("provider_name", req.Provider.Name), + attribute.String("applier_kind", applierKindOf(req)), + attribute.String("claim_namespace", req.Applier.ApplierClaim.Namespace), + attribute.String("claim_name", req.Applier.ApplierClaim.Name), + }, extra...) } // applierKindOf is the applier_kind label of a request's applier. diff --git a/internal/applier/dispatcher_test.go b/internal/applier/dispatcher_test.go index 75758728d..b28c794c2 100644 --- a/internal/applier/dispatcher_test.go +++ b/internal/applier/dispatcher_test.go @@ -345,6 +345,7 @@ func (f *dispatcherFixture) count(reason v1alpha3.ApplyReason, outcome string) i func (f *dispatcherFixture) countKind(kind string, reason v1alpha3.ApplyReason, outcome string) int64 { value, _ := telemetry.CollectInt64Sum(f.reader, applyRequestsMetric, map[string]string{ "applier_kind": kind, "reason": string(reason), "outcome": outcome, + "gitbranch_namespace": testGitBranch.Namespace, "gitbranch_name": testGitBranch.Name, }) return value } @@ -1209,8 +1210,14 @@ func TestDispatcher_AnApplyOfAnotherRevisionIsCounted(t *testing.T) { assert.Equal(t, git.ApplySucceeded, res.Outcome) assert.Contains(t, res.Message, "not commit") mismatches, _ := telemetry.CollectInt64Sum(f.reader, "gitopsreverser_apply_revision_mismatches_total", - map[string]string{"applier_kind": applierKindKustomization}) - assert.Equal(t, int64(1), mismatches) + map[string]string{ + "applier_kind": applierKindKustomization, + "gitbranch_namespace": testGitBranch.Namespace, + "gitbranch_name": testGitBranch.Name, + "claim_namespace": req.Applier.ApplierClaim.Namespace, + "claim_name": "checkout", + }) + assert.Equal(t, int64(1), mismatches, "counted under the GitBranch that asked and the claim it asked") } // TestApplyVerdict_ReadsTheReconcileThatHandledTheToken pins the outcome table. diff --git a/internal/attribution/route_health.go b/internal/attribution/route_health.go index 1326fc860..be8db6255 100644 --- a/internal/attribution/route_health.go +++ b/internal/attribution/route_health.go @@ -14,6 +14,7 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/event" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" @@ -49,6 +50,12 @@ type RouteHealth struct { absent map[string]int warned map[string]bool + // declared is the route each live KubeProvider carries, as its reconcile last saw it. The audit + // ingress labels its census by route only when some provider declares it: the route is a path + // segment or an annotation the producer chose, and a label taken from it unchecked would be + // bounded by nothing this operator controls. + declared map[types.NamespacedName]string + // delivered and transportFailing are PROCESS-WIDE, deliberately not keyed by route: an audit // request reaching this operator at all, and the fact transport being able to accept a write, // are properties of the pipeline rather than of any one route. They exist so a provider with no @@ -135,6 +142,46 @@ func notifyFirstFact(ch chan event.GenericEvent, route string) { } } +// DeclareRoute records the route one KubeProvider carries, replacing what it carried before. +func (h *RouteHealth) DeclareRoute(provider types.NamespacedName, route string) { + if h == nil { + return + } + h.mu.Lock() + defer h.mu.Unlock() + if h.declared == nil { + h.declared = map[types.NamespacedName]string{} + } + h.declared[provider] = route +} + +// ForgetRoute drops one KubeProvider's route: it was deleted, or no longer uses its route. +func (h *RouteHealth) ForgetRoute(provider types.NamespacedName) { + if h == nil { + return + } + h.mu.Lock() + defer h.mu.Unlock() + delete(h.declared, provider) +} + +// RouteDeclared reports whether some live KubeProvider carries route. It is a scan over the +// providers rather than an index: there are as many as someone wrote, and the caller asks once per +// request, or once per event on the shared endpoint. +func (h *RouteHealth) RouteDeclared(route string) bool { + if h == nil || route == "" { + return false + } + h.mu.Lock() + defer h.mu.Unlock() + for _, declared := range h.declared { + if declared == route { + return true + } + } + return false +} + // RecordAuditDelivery notes that an audit request carrying events reached this operator. It says // nothing about routes: the apiserver posting anywhere is what it establishes, which is the // difference between "nobody is sending us audit" and "your route is not the one being sent". diff --git a/internal/audit/outcome/outcome.go b/internal/audit/outcome/outcome.go index 726a02d76..dbaac22be 100644 --- a/internal/audit/outcome/outcome.go +++ b/internal/audit/outcome/outcome.go @@ -120,15 +120,28 @@ func (o Outcome) Category() Category { } } +// The audit_route values that are not a route. A route is a label only once a KubeProvider declares +// it, because the producer chooses it: unchecked, the label would be bounded by nothing this +// operator controls. +const ( + // RouteUnrecognized is an event posted under a route no KubeProvider carries. + RouteUnrecognized = "unrecognized" + // RouteNone is an event on the shared endpoint that names no route at all. + RouteNone = "none" +) + // Record increments gitopsreverser_audit_events_total once for the event's -// outcome, labelled {outcome, category, group, version, resource, verb}. It is -// nil-guarded so it is a no-op when telemetry is not initialized (unit tests). -func Record(ctx context.Context, event *auditv1.Event, o Outcome) { +// outcome, labelled {audit_route, outcome, category, group, version, resource, +// verb}. auditRoute is the caller's already-bounded label value: a declared +// route, RouteUnrecognized or RouteNone. It is nil-guarded so it is a no-op when +// telemetry is not initialized (unit tests). +func Record(ctx context.Context, event *auditv1.Event, o Outcome, auditRoute string) { if telemetry.AuditEventsTotal == nil { return } group, version, resource := gvrParts(event) telemetry.AuditEventsTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("audit_route", auditRoute), attribute.String("outcome", string(o)), attribute.String("category", string(o.Category())), attribute.String("group", group), diff --git a/internal/audit/outcome/outcome_test.go b/internal/audit/outcome/outcome_test.go index 2826b7fe9..a89a440be 100644 --- a/internal/audit/outcome/outcome_test.go +++ b/internal/audit/outcome/outcome_test.go @@ -52,7 +52,7 @@ func TestRecordNilGuard(t *testing.T) { // This test must stay ahead of the tests below: InitTestExporter wires the global // instrument permanently, so the nil branch is only reachable before the first call. assert.NotPanics(t, func() { - Record(context.Background(), nil, Queued) + Record(context.Background(), nil, Queued, RouteNone) }) } @@ -165,7 +165,7 @@ func TestVerb_NilEventAndPassthrough(t *testing.T) { } // TestRecord_EmitsLabelledSample covers the recording branch of Record (telemetry -// initialized) and pins the six labels it stamps, including that category is derived from +// initialized) and pins the seven labels it stamps, including that category is derived from // the outcome rather than passed in — an alert on category="error" depends on that. func TestRecord_EmitsLabelledSample(t *testing.T) { reader, err := telemetry.InitTestExporter() @@ -175,15 +175,16 @@ func TestRecord_EmitsLabelledSample(t *testing.T) { Record(ctx, &auditv1.Event{ Verb: "update", ObjectRef: &auditv1.ObjectReference{APIGroup: "apps", APIVersion: "apps/v1", Resource: "deployments"}, - }, WriteError) + }, WriteError, "team-a.prod") count, ok := telemetry.CollectInt64Sum(reader, auditEventsMetric, map[string]string{ - "outcome": string(WriteError), - "category": string(Error), - "group": "apps", - "version": "v1", - "resource": "deployments", - "verb": "update", + "audit_route": "team-a.prod", + "outcome": string(WriteError), + "category": string(Error), + "group": "apps", + "version": "v1", + "resource": "deployments", + "verb": "update", }) require.True(t, ok, "expected a sample for the recorded write_error outcome") assert.Equal(t, int64(1), count) @@ -196,7 +197,7 @@ func TestRecord_NilEventStillRecords(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) - Record(context.Background(), nil, NilEvent) + Record(context.Background(), nil, NilEvent, RouteNone) count, ok := telemetry.CollectInt64Sum(reader, auditEventsMetric, map[string]string{ "outcome": string(NilEvent), diff --git a/internal/controller/fluxclaim_hold.go b/internal/controller/fluxclaim_hold.go index 2499e45f8..b1157ca33 100644 --- a/internal/controller/fluxclaim_hold.go +++ b/internal/controller/fluxclaim_hold.go @@ -911,7 +911,7 @@ func (r *ApplierClaimReconciler) event(object client.Object, eventType, reason, } } -// applier_kind label values of claimed_resource_reconciled_total, as apply_requests_total spells them. +// applier_kind label values of apply_claimed_resource_reconciled_total, as apply_requests_total spells them. const ( applierKindKustomization = "kustomization" applierKindApplication = "application" diff --git a/internal/controller/fluxclaim_hold_test.go b/internal/controller/fluxclaim_hold_test.go index e11b74340..e2acdd359 100644 --- a/internal/controller/fluxclaim_hold_test.go +++ b/internal/controller/fluxclaim_hold_test.go @@ -159,9 +159,9 @@ func TestFluxClaim_AnExternalResumeIsSuspendedAgain(t *testing.T) { assert.Len(t, f.sim.Applied(), 2, "each resume applied once, unchecked; that is what the record says") } -// claimedResourceReconciledCount reads claimed_resource_reconciled_total for the test claim. +// claimedResourceReconciledCount reads apply_claimed_resource_reconciled_total for the test claim. func claimedResourceReconciledCount(reader *sdkmetric.ManualReader, applierKind string) (int64, bool) { - return telemetry.CollectInt64Sum(reader, "gitopsreverser_claimed_resource_reconciled_total", + return telemetry.CollectInt64Sum(reader, "gitopsreverser_apply_claimed_resource_reconciled_total", map[string]string{ "applier_kind": applierKind, "claim_namespace": claimKey.Namespace, "claim_name": claimKey.Name, }) diff --git a/internal/controller/gittarget_branch_target_metric_test.go b/internal/controller/gittarget_branch_target_metric_test.go index 9fa599c4f..731b0be0b 100644 --- a/internal/controller/gittarget_branch_target_metric_test.go +++ b/internal/controller/gittarget_branch_target_metric_test.go @@ -29,8 +29,8 @@ func TestGitTargetCleanup_ForgetsTheBranchTargetMapping(t *testing.T) { require.NoError(t, err) defer telemetry.ForgetBranchTarget("team-a", "sibling") - telemetry.RecordBranchTarget("team-a", "mirror", "team-a", "acme", "main", "default") - telemetry.RecordBranchTarget("team-a", "sibling", "team-a", "acme", "main", "default") + telemetry.RecordBranchTarget("team-a", "mirror", "team-a", "acme-main", "default") + telemetry.RecordBranchTarget("team-a", "sibling", "team-a", "acme-main", "default") _, ok := telemetry.CollectInt64Sum(reader, branchTargetsMetricName, map[string]string{ "gittarget_namespace": "team-a", diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 9328c1442..eadb9996a 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -305,15 +305,11 @@ func (r *GitTargetReconciler) publishAheadOfGates( ) { layout, scanned := r.observeLayout(target) publishLayout(st, target, layout, scanned) - // The provider labels join git_pushes_total, which the worker labels with the provider it - // pushes through: the GitBranch's, which may live in another namespace than the GitBranch. - provider := dest.providerKey() - if provider.Name == "" { - provider.Namespace = dest.key.Namespace - } + // The GitBranch labels join git_pushes_total, which the worker labels with the GitBranch it + // writes for. They come from spec.gitBranchRef alone, so a GitBranch that does not exist yet is + // still named. telemetry.RecordBranchTarget( - target.Namespace, target.Name, provider.Namespace, provider.Name, dest.branchName(), - target.SourceCluster()) + target.Namespace, target.Name, dest.key.Namespace, dest.key.Name, target.SourceCluster()) } // convergeAsSuspended gives a suspended GitTarget the converged outcome, under its own reason. diff --git a/internal/controller/kubeprovider_audit_facts.go b/internal/controller/kubeprovider_audit_facts.go index 1d7c27e46..f31e2cd2f 100644 --- a/internal/controller/kubeprovider_audit_facts.go +++ b/internal/controller/kubeprovider_audit_facts.go @@ -8,6 +8,7 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -44,6 +45,10 @@ type AuditRouteFacts interface { // TransportName is the configured transport ("redis" or "memory"), used only to point the // reader at the right thing while publishes are failing. Empty yields neutral wording. TransportName() string + // DeclareRoute and ForgetRoute keep the set of routes some KubeProvider carries, which bounds + // the audit_route label on the audit census. + DeclareRoute(provider types.NamespacedName, route string) + ForgetRoute(provider types.NamespacedName) } // applyAuditFactsCondition writes the one-way AuditFactsReceived latch. diff --git a/internal/controller/kubeprovider_audit_facts_test.go b/internal/controller/kubeprovider_audit_facts_test.go index 50b7b6a18..d7ff59f3f 100644 --- a/internal/controller/kubeprovider_audit_facts_test.go +++ b/internal/controller/kubeprovider_audit_facts_test.go @@ -360,3 +360,19 @@ func TestKubeProviderAuditFacts_RouteChangeStartsTheLatchOver(t *testing.T) { assert.Equal(t, metav1.ConditionTrue, findCondition(got.Status.Conditions, KubeProviderConditionAuditFactsReceived).Status) } + +// TestKubeProviderAuditFacts_DeclaresItsRouteUntilDeleted: the reconcile publishes the route the +// provider carries, which is what lets the audit census name it, and the NotFound reconcile after +// a delete takes it back. +func TestKubeProviderAuditFacts_DeclaresItsRouteUntilDeleted(t *testing.T) { + provider := inClusterKubeProvider("srcns-delegating") + health := &attribution.RouteHealth{} + + r, got := reconcileKubeProviderOnce(t, provider, health) + assert.True(t, health.RouteDeclared("team-a.srcns-delegating")) + + require.NoError(t, r.Delete(context.Background(), &got)) + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: provider.Key()}) + require.NoError(t, err) + assert.False(t, health.RouteDeclared("team-a.srcns-delegating"), "a deleted provider declares nothing") +} diff --git a/internal/controller/kubeprovider_controller.go b/internal/controller/kubeprovider_controller.go index 4e0688fca..e6b8bc0c0 100644 --- a/internal/controller/kubeprovider_controller.go +++ b/internal/controller/kubeprovider_controller.go @@ -86,18 +86,28 @@ func (r *KubeProviderReconciler) Reconcile(ctx context.Context, req ctrl.Request // value: a gauge that keeps reporting Ready=False for a deleted object is a permanent // false positive that trains people to ignore the panel. telemetry.ForgetResourceConditions(conditionKindKubeProvider, req.Namespace, req.Name) + r.forgetAuditRoute(req.NamespacedName) return ctrl.Result{}, nil } log.Error(err, "unable to fetch KubeProvider", "kubeProvider", req.NamespacedName) return ctrl.Result{}, err } if !provider.DeletionTimestamp.IsZero() { + // A provider being deleted carries no route (v1alpha3.AuditRouteHolder). + r.forgetAuditRoute(req.NamespacedName) return ctrl.Result{}, nil } return r.reconcileKubeProvider(ctx, log, &provider) } +// forgetAuditRoute drops a provider's route from the set the audit census labels by. +func (r *KubeProviderReconciler) forgetAuditRoute(provider k8stypes.NamespacedName) { + if r.AuditFacts != nil { + r.AuditFacts.ForgetRoute(provider) + } +} + // reconcileKubeProvider performs the main validation logic. func (r *KubeProviderReconciler) reconcileKubeProvider( ctx context.Context, @@ -132,6 +142,11 @@ func (r *KubeProviderReconciler) reconcileKubeProvider( if !route.Allowed { rd.stalled(route.Reason, route.Message) } + // Declared whether or not this provider holds it: facts posted under a route are labelled by it + // as long as any provider names it, and the holder declares it too. + if r.AuditFacts != nil { + r.AuditFacts.DeclareRoute(provider.Key(), provider.AuditRoute()) + } st.set(KubeProviderConditionValidated, validated, reason, message) // Written after the readiness trio and deliberately not contributing to it: an audit route // nobody posts under costs the commit AUTHOR, never the mirror, so it must not make a working diff --git a/internal/git/applier_claim_test.go b/internal/git/applier_claim_test.go index 1a82e050b..b4a986ee5 100644 --- a/internal/git/applier_claim_test.go +++ b/internal/git/applier_claim_test.go @@ -368,8 +368,7 @@ func TestCloseWindowForPushHint_PublishesTheOpenWindowNow(t *testing.T) { t.Cleanup(loop.stopTimers) loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("edited", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("edited", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow, "an hour-long window is still open") diff --git a/internal/git/applier_echoes_test.go b/internal/git/applier_echoes_test.go index 086dbf170..aa2c9cd8d 100644 --- a/internal/git/applier_echoes_test.go +++ b/internal/git/applier_echoes_test.go @@ -133,7 +133,7 @@ func splitLines(s string) []string { // writeWindow sends one window of live writes, closed at once (the loop's window is zero). func (f *echoFixture) writeWindow(events ...Event) { - f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: events, CommitMode: CommitModePerEvent}}) + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: events}}) } // TestApplierEchoes_AnUnchangedObjectIsNoAction is F2 for a window, without timing luck: the write diff --git a/internal/git/apply_catchup_test.go b/internal/git/apply_catchup_test.go index 0c51da5ce..d68e93b10 100644 --- a/internal/git/apply_catchup_test.go +++ b/internal/git/apply_catchup_test.go @@ -78,16 +78,16 @@ func TestCatchUp_AnAPICreateIsAdoptedEvenAtTheTip(t *testing.T) { require.Empty(t, requester.taken()) ownApply := createdEvent("alpha", tc.manager) - loop.handleLiveEvents(&WriteRequest{Events: []Event{ownApply}, CommitMode: CommitModePerEvent}) + loop.handleLiveEvents(&WriteRequest{Events: []Event{ownApply}}) owned := createdEvent("alpha", "kubectl-create") owned.AppliedBy = AppliedBy{Kind: configv1alpha3.ApplierKindKustomization, Namespace: "x", Name: "other"} - loop.handleLiveEvents(&WriteRequest{Events: []Event{owned}, CommitMode: CommitModePerEvent}) + loop.handleLiveEvents(&WriteRequest{Events: []Event{owned}}) loop.openWindow = nil loop.serviceAppliers() require.Empty(t, requester.taken(), "neither is an API create of the folder's own objects") loop.handleLiveEvents(&WriteRequest{ - Events: []Event{createdEvent("alpha", "kubectl-create")}, CommitMode: CommitModePerEvent, + Events: []Event{createdEvent("alpha", "kubectl-create")}, }) loop.openWindow = nil loop.serviceAppliers() diff --git a/internal/git/apply_epoch.go b/internal/git/apply_epoch.go index 406315d69..b92c3c9e5 100644 --- a/internal/git/apply_epoch.go +++ b/internal/git/apply_epoch.go @@ -38,7 +38,7 @@ import ( // discarded: applierRetryMax, so an applier at full backoff gets at least one more attempt first. const snapshotHoldLimit = applierRetryMax -// Stale snapshot outcomes, the outcome label of stale_snapshots_total. +// Stale snapshot outcomes, the outcome label of apply_stale_snapshots_total. const ( staleSnapshotDiscarded = "discarded" staleSnapshotUsed = "used" @@ -234,5 +234,5 @@ func (l *branchWorkerEventLoop) recordStaleSnapshot(outcome string) { return } telemetry.StaleSnapshotsTotal.Add(context.WithoutCancel(l.w.ctx), 1, - metric.WithAttributes(l.w.providerAttrs(attribute.String("outcome", outcome))...)) + metric.WithAttributes(l.w.gitBranchAttrs(attribute.String("outcome", outcome))...)) } diff --git a/internal/git/apply_ownership.go b/internal/git/apply_ownership.go index 650eb4478..c3e0d1595 100644 --- a/internal/git/apply_ownership.go +++ b/internal/git/apply_ownership.go @@ -179,7 +179,7 @@ const ownershipLogLimit = 1024 func (l *branchWorkerEventLoop) recordOwnershipConflict(target, object string, applier ApplierRef, owner AppliedBy) { if telemetry.ApplyOwnershipConflictsTotal != nil { telemetry.ApplyOwnershipConflictsTotal.Add(context.WithoutCancel(l.w.ctx), 1, - metric.WithAttributes(l.w.providerAttrs()...)) + metric.WithAttributes(l.w.gitBranchAttrs()...)) } key := target + "|" + object if _, logged := l.ownershipLogged[key]; logged { diff --git a/internal/git/apply_ownership_test.go b/internal/git/apply_ownership_test.go index 6594d7661..8a1fd896e 100644 --- a/internal/git/apply_ownership_test.go +++ b/internal/git/apply_ownership_test.go @@ -91,7 +91,7 @@ func TestOwnership_ALiveWriteOfAnotherAppliersObjectIsNotMirrored(t *testing.T) event := windowEvent("alpha", "2", "kubectl-edit") event.AppliedBy = other - loop.handleLiveEvents(&WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}) + loop.handleLiveEvents(&WriteRequest{Events: []Event{event}}) assert.Nil(t, loop.openWindow, "the object another applier owns opened no window") } diff --git a/internal/git/apply_window.go b/internal/git/apply_window.go index 0cb4cc769..565d36da3 100644 --- a/internal/git/apply_window.go +++ b/internal/git/apply_window.go @@ -316,7 +316,7 @@ func (l *branchWorkerEventLoop) recordApplyEcho(outcome string) { return } telemetry.ApplyEchoesTotal.Add(context.WithoutCancel(l.w.ctx), 1, - metric.WithAttributes(l.w.providerAttrs(attribute.String("outcome", outcome))...)) + metric.WithAttributes(l.w.gitBranchAttrs(attribute.String("outcome", outcome))...)) } // reportApplyWindow logs, once, how many of the applier's writes its window kept out of Git: what it @@ -372,5 +372,5 @@ func (l *branchWorkerEventLoop) recordApplyWindowCollision( return } telemetry.ApplyWindowCollisionsTotal.Add(context.WithoutCancel(w.ctx), 1, - metric.WithAttributes(w.providerAttrs(attribute.String("outcome", outcome))...)) + metric.WithAttributes(w.gitBranchAttrs(attribute.String("outcome", outcome))...)) } diff --git a/internal/git/apply_window_test.go b/internal/git/apply_window_test.go index 42083a324..38086a9b7 100644 --- a/internal/git/apply_window_test.go +++ b/internal/git/apply_window_test.go @@ -124,7 +124,7 @@ func TestApplyWindow_OnlyAWithheldWriteIsKeptFromTheCommitWindow(t *testing.T) { loop.handleLiveEvents(&WriteRequest{Events: []Event{ windowEvent("alpha", "3", "kubectl-edit"), windowEvent("alpha", "2", FluxFieldManager), - }, CommitMode: CommitModePerEvent}) + }}) require.NotNil(t, loop.openWindow) require.Len(t, loop.openWindow.pathToEvent, 1) diff --git a/internal/git/attribution_outcome_test.go b/internal/git/attribution_outcome_test.go index 6bf9533ca..a7a581dfa 100644 --- a/internal/git/attribution_outcome_test.go +++ b/internal/git/attribution_outcome_test.go @@ -113,8 +113,8 @@ func TestAuthorKind_ClassifiesAllFourKinds(t *testing.T) { want: authorKindUnresolved, }, { - name: "an atomic write never attempts attribution", - write: PendingWrite{Kind: PendingWriteAtomic}, + name: "an empty write never attempts attribution", + write: PendingWrite{Kind: PendingWriteCommit}, want: authorKindCommitter, }, } { diff --git a/internal/git/barrier.go b/internal/git/barrier.go index 163e56e17..042f2ec35 100644 --- a/internal/git/barrier.go +++ b/internal/git/barrier.go @@ -26,7 +26,7 @@ import ( // // The hold is approved; the silence was the defect. Every applier the barrier holds a reason back // for is logged at Info when the hold begins, changes reason or lifts, and counted in the -// appliers_held gauge. +// apply_appliers_held gauge. // barrierHold says why the barrier holds the applier back, leaving out its folders // (missingFolders), or "" when it does not. @@ -123,7 +123,7 @@ func (l *branchWorkerEventLoop) noteHeld(state *applierState, held string) { } // publishHeld publishes how many of this branch's appliers the barrier holds a reason back for, for -// the appliers_held gauge. +// the apply_appliers_held gauge. func (l *branchWorkerEventLoop) publishHeld() { var held int64 for _, state := range l.appliers { diff --git a/internal/git/bootstrapped_repo_template_test.go b/internal/git/bootstrapped_repo_template_test.go index f7b13281f..47e11c890 100644 --- a/internal/git/bootstrapped_repo_template_test.go +++ b/internal/git/bootstrapped_repo_template_test.go @@ -3,9 +3,12 @@ package git import ( + "os" "path" + "path/filepath" "testing" + gogit "github.com/go-git/go-git/v6" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,3 +36,57 @@ func TestRenderSOPSBootstrapTemplate_MissingRecipients(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "missing age recipients") } + +// bootstrapRepo is an empty local repository to stage a folder's bootstrap files into. +func bootstrapRepo(t *testing.T) (*gogit.Repository, string) { + t.Helper() + dir := t.TempDir() + repo, err := gogit.PlainInit(dir, false) + require.NoError(t, err) + return repo, dir +} + +// The folder bootstrap the write path stages before a GitTarget's first document: a README and the +// commented .gittargetignore always, a .sops.yaml only once encryption resolves recipients (it is +// added on a later write, not withheld forever), every recipient rendered, and a file somebody +// already wrote never overwritten. Staging twice is a no-op. +func TestEnsureBootstrapTemplateInPath_StagesTheFolderFiles(t *testing.T) { + repo, dir := bootstrapRepo(t) + folder := filepath.Join(dir, "clusters/dev") + require.NoError(t, os.MkdirAll(folder, 0o750)) + custom := "# custom readme\n" + require.NoError(t, os.WriteFile(filepath.Join(folder, "README.md"), []byte(custom), 0o600)) + + plain := buildBootstrapOptions(nil) + require.NoError(t, ensureBootstrapTemplateInPath(repo, "clusters/dev", plain)) + require.NoError(t, ensureBootstrapTemplateInPath(repo, "clusters/dev", plain)) + + readme, err := os.ReadFile(filepath.Join(folder, "README.md")) + require.NoError(t, err) + assert.Equal(t, custom, string(readme), "an existing file is never overwritten") + _, err = os.Stat(filepath.Join(folder, gitTargetIgnoreFileName)) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(folder, sopsConfigFileName)) + assert.True(t, os.IsNotExist(err), "no .sops.yaml before encryption resolves recipients") + + recipients := []string{ + "age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq7k8m6", + "age1yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyv0r2a", + } + encrypted := buildBootstrapOptions(&ResolvedEncryptionConfig{AgeRecipients: recipients}) + require.NoError(t, ensureBootstrapTemplateInPath(repo, "clusters/dev", encrypted)) + + sops, err := os.ReadFile(filepath.Join(folder, sopsConfigFileName)) + require.NoError(t, err, "encryption configured later adds the .sops.yaml") + for _, recipient := range recipients { + assert.Contains(t, string(sops), recipient) + } +} + +// A disabled bootstrap stages nothing at all. +func TestEnsureBootstrapTemplateInPath_DisabledStagesNothing(t *testing.T) { + repo, dir := bootstrapRepo(t) + require.NoError(t, ensureBootstrapTemplateInPath(repo, "clusters/dev", pathBootstrapOptions{})) + _, err := os.Stat(filepath.Join(dir, "clusters/dev")) + assert.True(t, os.IsNotExist(err)) +} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 54cdc303c..4843d5f10 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -154,7 +154,7 @@ type BranchWorker struct { // loop last published it, for the apply_operations_unresolved gauge. unresolvedOperations atomic.Int64 // heldAppliers is how many of this branch's appliers the barrier holds a reason back for, as the - // loop last published it, for the appliers_held gauge (barrier.go). + // loop last published it, for the apply_appliers_held gauge (barrier.go). heldAppliers atomic.Int64 // applyEpoch, appliersAhead and hasAppliers are the loop's ApplyState for the watch side, which // reads it when a snapshot's collection starts (apply_epoch.go). @@ -407,7 +407,6 @@ const ( windowFinalizeReasonMaxDuration windowFinalizeReason = "max-duration" windowFinalizeReasonAttachNext windowFinalizeReason = "attach-next" windowFinalizeReasonResyncBeforeApply windowFinalizeReason = "resync-before-apply" - windowFinalizeReasonAtomicBeforeApply windowFinalizeReason = "atomic-before-apply" windowFinalizeReasonIdentityChange windowFinalizeReason = "author-or-target-change" windowFinalizeReasonBufferLimit windowFinalizeReason = "buffer-limit" windowFinalizeReasonShutdown windowFinalizeReason = "shutdown" @@ -604,11 +603,7 @@ func (w *BranchWorker) Stop() { // dropped, so a caller advancing a durable watch cursor past this event must not treat // the drop as success (see reconcile.GitTargetEventStream.OnWatchEvent). func (w *BranchWorker) Enqueue(event Event) bool { - return w.enqueueRequest(&WriteRequest{ - Events: []Event{event}, - CommitMode: CommitModePerEvent, - CommitMessage: "", - }) + return w.enqueueRequest(&WriteRequest{Events: []Event{event}}) } // EnqueueWithdraw asks the loop to cancel a CommitRequest it has not acted on. It never blocks: a @@ -840,7 +835,7 @@ func (w *BranchWorker) markResyncTailForWriteLocked(request *WriteRequest) { continue } for i := range request.Events { - namespace, name := writeEventTarget(request, i) + namespace, name := request.Events[i].GitTargetNamespace, request.Events[i].GitTargetName if key.namespace != namespace || key.name != name { continue } @@ -872,16 +867,6 @@ func (w *BranchWorker) markResyncTailForResyncLocked(request *ResyncRequest, key } } -// writeEventTarget resolves the GitTarget one event of a write belongs to: the event's -// own fields when set, falling back to the request's for the atomic paths that carry the -// target once for the whole request. -func writeEventTarget(request *WriteRequest, i int) (string, string) { - if request.Events[i].GitTargetName != "" { - return request.Events[i].GitTargetNamespace, request.Events[i].GitTargetName - } - return request.GitTargetNamespace, request.GitTargetName -} - // markResyncTailForTargetLocked records the same boundary for every pending resync of one // GitTarget, regardless of scope. It is used for queued items that carry no resource // identity of their own. The caller must hold pendingResyncsMu. @@ -951,7 +936,7 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { w.recordQueueRefusal(queueRefusalWrite) w.Log.V(1).Info("Worker is stopping, request refused; the producer keeps it to redeliver", "events", len(request.Events), - "gitTarget", request.GitTargetName) + "gitTarget", request.gitTarget()) return false } if !w.admitPayload(item.charge) { @@ -959,7 +944,7 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { w.releaseInflightItem() w.recordQueueRefusal(queueRefusalWrite) w.Log.V(1).Info("Write refused while the branch has paused intake; "+ - "the producer keeps it to redeliver", "events", len(request.Events), "gitTarget", request.GitTargetName) + "the producer keeps it to redeliver", "events", len(request.Events), "gitTarget", request.gitTarget()) return false } w.markResyncTailForWriteLocked(request) @@ -968,8 +953,7 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { w.pendingResyncsMu.Unlock() w.Log.V(1).Info("Write request enqueued", "events", len(request.Events), - "mode", request.CommitMode, - "gitTarget", request.GitTargetName, + "gitTarget", request.gitTarget(), "sourceCollection", sourceCollectionForLog(request.sourceCollection())) // Nothing publishes depth here: the gauge reads inflightItems at scrape // time, so an enqueue is visible to the next scrape whether or not the @@ -985,8 +969,7 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { // one may be one COLLECTION of one GitTarget. w.Log.Error(nil, "Event queue full, request refused; its producer offers it again", "events", len(request.Events), - "mode", request.CommitMode, - "gitTarget", request.GitTargetName, + "gitTarget", request.gitTarget(), "sourceCollection", sourceCollectionForLog(request.sourceCollection())) return false } @@ -1006,136 +989,6 @@ func (w *BranchWorker) queueDepth() int64 { return depth } -// EnsurePathBootstrapped prepares bootstrap templates locally for a path. -// Existing files are preserved, and only missing template files are added. -// The files are staged in the local worktree but never committed or pushed here. -func (w *BranchWorker) EnsurePathBootstrapped(path, targetName, targetNamespace string) error { - w.repoMu.Lock() - defer w.repoMu.Unlock() - - ctx := w.ctx - if ctx == nil { - ctx = context.Background() - } - - normalizedPath, err := normalizeBootstrapPath(path) - if err != nil { - return err - } - if err := validateBootstrapTarget(targetName, targetNamespace); err != nil { - return err - } - - bootstrapOptions, err := w.resolveBootstrapOptions(ctx, targetName, targetNamespace) - if err != nil { - return err - } - - repoPath, err := w.prepareBootstrapRepository(ctx) - if err != nil { - return err - } - - if err := w.bootstrapPathIfNeeded(repoPath, normalizedPath, bootstrapOptions); err != nil { - return err - } - - w.Log.Info("Prepared bootstrap files in worktree", "path", normalizedPath) - return nil -} - -func normalizeBootstrapPath(path string) (string, error) { - normalizedPath := sanitizePath(path) - if strings.TrimSpace(path) != "" && normalizedPath == "" { - return "", fmt.Errorf("invalid path %q", path) - } - return normalizedPath, nil -} - -func validateBootstrapTarget(targetName, targetNamespace string) error { - if strings.TrimSpace(targetName) == "" || strings.TrimSpace(targetNamespace) == "" { - return errors.New("target name and namespace must be set for path bootstrap") - } - return nil -} - -func (w *BranchWorker) resolveBootstrapOptions( - ctx context.Context, - targetName string, - targetNamespace string, -) (pathBootstrapOptions, error) { - targetKey := types.NamespacedName{Name: targetName, Namespace: targetNamespace} - var target configv1alpha3.GitTarget - if err := w.Client.Get(ctx, targetKey, &target); err != nil { - return pathBootstrapOptions{}, fmt.Errorf("failed to get GitTarget %s: %w", targetKey.String(), err) - } - - encryptionConfig, err := ResolveTargetEncryption(ctx, w.Client, &target) - if err != nil { - w.Log.Error(err, "Skipping SOPS bootstrap due to invalid target encryption configuration", - "target", targetKey.String()) - return pathBootstrapOptions{Enabled: true}, nil - } - if encryptionConfig == nil { - return pathBootstrapOptions{Enabled: true}, nil - } - - if len(encryptionConfig.AgeRecipients) == 0 { - w.Log.Info("Skipping SOPS bootstrap due to missing resolved age recipients", - "target", targetKey.String()) - return pathBootstrapOptions{Enabled: true}, nil - } - - return pathBootstrapOptions{ - Enabled: true, - IncludeSOPSConfig: true, - TemplateData: bootstrapTemplateData{ - AgeRecipients: encryptionConfig.AgeRecipients, - }, - }, nil -} - -func (w *BranchWorker) prepareBootstrapRepository( - ctx context.Context, -) (string, error) { - provider, err := w.getGitProvider(ctx) - if err != nil { - return "", fmt.Errorf("failed to get GitProvider: %w", err) - } - - auth, err := getAuthFromSecret(ctx, w.Client, provider, w.sshHostKeys, w.credentialPolicy) - if err != nil { - return "", fmt.Errorf("failed to get auth: %w", err) - } - - repoPath := w.repoPath() - w.recordFetch(fetchReasonBootstrap) - pullReport, err := w.prepareBranch(ctx, repoPath, auth) - if err != nil { - return "", fmt.Errorf("failed to prepare repository: %w", err) - } - w.updateBranchMetadataFromPullReport(pullReport) - - return repoPath, nil -} - -func (w *BranchWorker) bootstrapPathIfNeeded( - repoPath string, - normalizedPath string, - options pathBootstrapOptions, -) error { - repo, err := gogit.PlainOpen(repoPath) - if err != nil { - return fmt.Errorf("failed to open repository: %w", err) - } - - if err := ensureBootstrapTemplateInPath(repo, normalizedPath, options); err != nil { - return fmt.Errorf("failed to bootstrap path %q: %w", normalizedPath, err) - } - - return nil -} - // processEvents is the main event processing loop. // // The loop owns one live commit-shaped event window. A commit-window timer @@ -1172,7 +1025,7 @@ type branchWorkerEventLoop struct { defaultWindow commitWindowDefaults // openWindow holds the one live commit-shaped event window. It is finalized when its idle or - // max-duration timer runs out, and eagerly on author/target changes, atomic arrivals, byte-cap + // max-duration timer runs out, and eagerly on author/target changes, resync arrivals, byte-cap // trips, an attach: Next, and shutdown. openWindow *openWindow windowBytes int64 @@ -1430,14 +1283,6 @@ func (l *branchWorkerEventLoop) dispatch(item WorkItem) int64 { } l.handleRefreshRequest(item.Refresh) case item.Request == nil: - case item.Request.CommitMode == CommitModeAtomic: - targetName, targetNamespace := atomicRefusalTarget(item.Request) - if status, closed := l.w.renderFidelityClosed(targetName, targetNamespace); closed { - l.dropAtRenderFidelity(itypes.NewResourceReference(targetName, targetNamespace), status, - writeDroppedAtomic, "") - break - } - l.handleAtomicRequest(item.Request) default: l.handleLiveEvents(item.Request) } @@ -1536,7 +1381,7 @@ func (l *branchWorkerEventLoop) mirrors(event Event) bool { l.serviceHeld(target) status, closed := l.w.renderFidelityClosed(event.GitTargetName, event.GitTargetNamespace) if closed && status.State == RenderFidelityFalse { - l.dropAtRenderFidelity(target, status, writeDroppedEvent, event.Identifier.String()) + l.dropAtRenderFidelity(target, status, event.Identifier.String()) return false } if l.ownedElsewhere(event.GitTargetName, event.GitTargetNamespace, event.Identifier.String(), @@ -1576,22 +1421,6 @@ func (l *branchWorkerEventLoop) closeOrArmWindow() { l.resetCommitTimer(delay) } -// handleAtomicRequest decides one atomic write request. Atomic batches bypass the commit window, but -// not the pending writes: any open live work is finalized first so arrival order is preserved, then -// the atomic write joins the pending writes behind it. -func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { - l.finalizeOpenWindowWithReason(windowFinalizeReasonAtomicBeforeApply) - - pendingWrite, err := l.w.buildAtomicPendingWrite(l.w.ctx, request) - if err != nil { - l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) - l.w.Log.Error(err, "Failed to build atomic pending write", "events", len(request.Events)) - return - } - pendingWrite.origin.atomic = request - l.decide(*pendingWrite) -} - func (l *branchWorkerEventLoop) handleShutdown() { l.w.Log.Info("Handling shutdown, finalizing open window and pushing pending commits") l.finalizeOpenWindowWithReason(windowFinalizeReasonShutdown) @@ -1708,7 +1537,7 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali pendingWrite, err := l.w.buildGroupedPendingWrite(l.w.ctx, events) if err != nil { - l.w.recordCommitFailure(commitFailureKindWindow, commitFailureError) + l.w.recordCommitFailure(commitFailureError) l.w.Log.Error(err, "Failed to build pending write; dropping open window", "reason", string(reason), "windowAuthor", windowAuthor, @@ -2843,8 +2672,6 @@ const ( // of commitPendingWrites, and keeping them apart is what lets the steady-state claim be asserted // at zero without the assertion failing the first time a push fails. const ( - // fetchReasonBootstrap is the worker's first contact with the repository. - fetchReasonBootstrap = "bootstrap" // fetchReasonPublication is the fetch at the head of a publication cycle. fetchReasonPublication = "publication" // fetchReasonRecovery is a cycle that had to re-establish a base it could not trust. @@ -2874,13 +2701,10 @@ const ( fetchReasonForcedRecheck = "forced_recheck" ) -// Commit-failure kinds and reasons. A failure here is work that was accepted, routed, and then -// died before it could reach the remote: the window is dropped and its events are gone until the -// next resync re-derives them. +// Commit-failure reasons. A failure here is a window that was accepted, routed, and then died before +// it could reach the remote: the window is dropped and its events are gone until the next resync +// re-derives them. const ( - commitFailureKindWindow = "window" - commitFailureKindAtomic = "atomic" - // commitFailureRefused is a plan the acceptance gate or a write-boundary precondition // rejected. It is also surfaced as a GitTarget condition, but a condition is not a rate. commitFailureRefused = "refused" @@ -2928,10 +2752,10 @@ func (w *BranchWorker) publishCommitsForPush(pendingWrites []PendingWrite) { }]++ } for labels, count := range counts { - // Label by the recording BranchWorker's own identity {provider_namespace, - // provider_name, branch} plus author_kind and message_source. The prefixed key names - // avoid the reserved Prometheus pod-scrape labels `namespace`/`name`. - telemetry.GitCommitsTotal.Add(ctx, count, metric.WithAttributes(w.providerAttrs( + // Label by the recording BranchWorker's GitBranch {gitbranch_namespace, gitbranch_name} + // plus author_kind and message_source. The prefixed key names avoid the reserved + // Prometheus pod-scrape labels `namespace`/`name`. + telemetry.GitCommitsTotal.Add(ctx, count, metric.WithAttributes(w.gitBranchAttrs( attribute.String("author_kind", labels.authorKind), attribute.String("message_source", labels.messageSource), )...)) @@ -2950,11 +2774,11 @@ func (w *BranchWorker) recordPushOutcome(outcome string, started time.Time) { } if telemetry.GitPushesTotal != nil { telemetry.GitPushesTotal.Add(ctx, 1, - metric.WithAttributes(w.providerAttrs(attribute.String("outcome", outcome))...)) + metric.WithAttributes(w.gitBranchAttrs(attribute.String("outcome", outcome))...)) } if telemetry.GitPushDurationSeconds != nil { telemetry.GitPushDurationSeconds.Record(ctx, time.Since(started).Seconds(), - metric.WithAttributes(w.providerAttrs()...)) + metric.WithAttributes(w.gitBranchAttrs()...)) } } @@ -2968,7 +2792,7 @@ func (w *BranchWorker) recordPushRetry(reason string) { ctx = context.Background() } telemetry.GitPushRetriesTotal.Add(ctx, 1, - metric.WithAttributes(w.providerAttrs(attribute.String("reason", reason))...)) + metric.WithAttributes(w.gitBranchAttrs(attribute.String("reason", reason))...)) } // recordFetch counts one call that reaches the remote through SmartFetch. @@ -2985,28 +2809,34 @@ func (w *BranchWorker) recordFetch(reason string) { ctx = context.Background() } telemetry.GitFetchesTotal.Add(ctx, 1, - metric.WithAttributes(w.providerAttrs(attribute.String("reason", reason))...)) -} - -// providerAttrs is the {provider_namespace, provider_name, branch} identity every Git-side metric -// this worker records is labelled by. The keys are prefixed, never bare namespace/name: a -// Prometheus pod scrape with honor_labels=false overwrites a bare `namespace` attribute with the -// scraped pod's own, which silently breaks every per-provider selector. -func (w *BranchWorker) providerAttrs(extra ...attribute.KeyValue) []attribute.KeyValue { + metric.WithAttributes(w.gitBranchAttrs(attribute.String("reason", reason))...)) +} + +// gitBranchAttrs is the {gitbranch_namespace, gitbranch_name, provider_namespace, provider_name} +// identity every Git-side metric this worker records is labelled by. The GitBranch it writes for +// matches a series to that object's conditions and `kubectl get gitbranch`; its GitProvider groups +// the branches one credential writes, which is how an expired credential shows. The provider adds +// no series: it is immutable on the GitBranch, and a worker serves one GitBranch. spec.branch is +// left off for the same reason and because the GitBranch already names it. The keys are prefixed, +// never bare namespace/name: a Prometheus pod scrape with honor_labels=false renames a bare +// `namespace` to exported_namespace and puts the scraped pod's own in its place, which silently +// breaks every per-branch selector. +func (w *BranchWorker) gitBranchAttrs(extra ...attribute.KeyValue) []attribute.KeyValue { return append([]attribute.KeyValue{ + attribute.String("gitbranch_namespace", w.GitBranch.Namespace), + attribute.String("gitbranch_name", w.GitBranch.Name), attribute.String("provider_namespace", w.GitProviderNamespace), attribute.String("provider_name", w.GitProviderRef), - attribute.String("branch", w.Branch), }, extra...) } -// recordCommitFailure counts one window or request that died between routing and pushing: its +// recordCommitFailure counts one window that died between routing and pushing: its // events are lost until the next resync re-derives them. An unreachable remote is not one: the // decided write stays pending for the publication retry. // // `refused` and `error` need different people. A refusal is a Git path a human has to fix and will // not clear on its own; an error may be transient. -func (w *BranchWorker) recordCommitFailure(kind, reason string) { +func (w *BranchWorker) recordCommitFailure(reason string) { if telemetry.GitCommitFailuresTotal == nil { return } @@ -3014,10 +2844,8 @@ func (w *BranchWorker) recordCommitFailure(kind, reason string) { if ctx == nil { ctx = context.Background() } - telemetry.GitCommitFailuresTotal.Add(ctx, 1, metric.WithAttributes(w.providerAttrs( - attribute.String("kind", kind), - attribute.String("reason", reason), - )...)) + telemetry.GitCommitFailuresTotal.Add(ctx, 1, + metric.WithAttributes(w.gitBranchAttrs(attribute.String("reason", reason))...)) } // Commit window timer sources, reported as the commit_windows_total `timer_source` label. @@ -3042,8 +2870,6 @@ func (r windowFinalizeReason) metricLabel() string { return "buffer_limit" case windowFinalizeReasonResyncBeforeApply: return "resync_before_apply" - case windowFinalizeReasonAtomicBeforeApply: - return "atomic_before_apply" case windowFinalizeReasonShutdown: return "shutdown" case windowFinalizeReasonForeignCommit: @@ -3099,7 +2925,7 @@ func (w *BranchWorker) recordQueueRefusal(kind string) { ctx = context.Background() } telemetry.GitQueueRefusalsTotal.Add(ctx, 1, - metric.WithAttributes(w.providerAttrs(attribute.String("kind", kind))...)) + metric.WithAttributes(w.gitBranchAttrs(attribute.String("kind", kind))...)) } // getGitProvider fetches the GitProvider for this worker. @@ -3284,47 +3110,6 @@ func (w *BranchWorker) syncWithRemote(ctx context.Context, reason string) error return nil } -// ensureRepositoryInitialized ensures the worker's repository is cloned and ready. -func (w *BranchWorker) ensureRepositoryInitialized(ctx context.Context) error { - provider, err := w.getGitProvider(ctx) - if err != nil { - return fmt.Errorf("failed to get GitProvider: %w", err) - } - - repoPath := w.repoPath() - if w.shouldReadRetainedLocalRepository(repoPath) { - return nil - } - - auth, err := getAuthFromSecret(ctx, w.Client, provider, w.sshHostKeys, w.credentialPolicy) - if err != nil { - return fmt.Errorf("failed to get auth: %w", err) - } - - // Use new PrepareBranch abstraction - pullReport, err := w.prepareBranch(ctx, repoPath, auth) - if err != nil { - return fmt.Errorf("failed to prepare repository: %w", err) - } - - // Update metadata from pull report - w.updateBranchMetadataFromPullReport(pullReport) - - return nil -} - -func (w *BranchWorker) shouldReadRetainedLocalRepository(repoPath string) bool { - if !w.hasUnpushedWork.Load() && w.inflightItems.Load() == 0 { - return false - } - - if _, err := gogit.PlainOpen(repoPath); err != nil { - return false - } - - return true -} - // updateBranchMetadataFromPullReport updates metadata from a PullReport. func (w *BranchWorker) updateBranchMetadataFromPullReport(report *PullReport) { // A branch the remote does not have is recorded as an observation with no revision, which is diff --git a/internal/git/branch_worker_empty_repo_test.go b/internal/git/branch_worker_empty_repo_test.go index 70612387d..3795235dd 100644 --- a/internal/git/branch_worker_empty_repo_test.go +++ b/internal/git/branch_worker_empty_repo_test.go @@ -152,7 +152,7 @@ func TestBranchWorker_AnEmptyRepositoryThatGainsABranchKeepsANoOpAbsent(t *testi deleted := configMapTargetEvent("never-written", "alice", newBranchTarget) deleted.Operation = "DELETE" deleted.Object = nil - f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}, CommitMode: CommitModePerEvent}}) + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}}}) require.Empty(t, f.loop.pendingWrites, "settled at once, even in the cooldown") simulateClientCommitOnDisk(t, f.repoDir, "main", "README.md", "main appeared\n") @@ -276,7 +276,7 @@ func noOpWrite(loop *branchWorkerEventLoop) { deleted := configMapTargetEvent("never-written", "alice", newBranchTarget) deleted.Operation = "DELETE" deleted.Object = nil - loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}, CommitMode: CommitModePerEvent}}) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}}}) } // §4.7 R5. With spec.parentBranch omitted the parent is the default branch as last discovered. A diff --git a/internal/git/branch_worker_fetch_metric_test.go b/internal/git/branch_worker_fetch_metric_test.go index d2981f596..f564e2013 100644 --- a/internal/git/branch_worker_fetch_metric_test.go +++ b/internal/git/branch_worker_fetch_metric_test.go @@ -33,10 +33,9 @@ const fetchesTotalMetric = "gitopsreverser_git_fetches_total" // fetchLabels is the identity a ledger fixture's worker records under, narrowed to one reason. func fetchLabels(worker *BranchWorker, reason string) map[string]string { return map[string]string{ - "provider_namespace": worker.GitProviderNamespace, - "provider_name": worker.GitProviderRef, - "branch": worker.Branch, - "reason": reason, + "gitbranch_namespace": worker.GitBranch.Namespace, + "gitbranch_name": worker.GitBranch.Name, + "reason": reason, } } @@ -192,20 +191,15 @@ func TestGitFetchesTotal_ProbeThenResetWhenTheRemoteDidMove(t *testing.T) { "the probe confirmed movement, so the reset that follows is contention") } -// TestGitFetchesTotal_ForcedRecheckAndBootstrap covers the two reasons that must keep counting -// after the flip: a worker's first contact with the repository, and somebody asking it to look at -// Git now. -func TestGitFetchesTotal_ForcedRecheckAndBootstrap(t *testing.T) { +// TestGitFetchesTotal_ForcedRecheck covers the reason that must keep counting after the flip: +// somebody asking the worker to look at Git now. +func TestGitFetchesTotal_ForcedRecheck(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) f := newLedgerFixture(t, "metric-recheck", true) f.createLedgerTarget("team-a", nil) - - require.NoError(t, f.worker.EnsurePathBootstrapped("team-a", "target-a", "default")) - assert.Equal(t, int64(1), fetchCount(t, reader, f.worker, fetchReasonBootstrap), - "the bootstrap is instrumented at prepareBootstrapRepository. NOTE: its only caller is\n"+ - "EnsurePathBootstrapped, which has no production caller, so this series is test-only today") + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) err = f.worker.syncWithRemote(f.worker.ctx, fetchReasonForcedRecheck) require.NoError(t, err) @@ -239,7 +233,6 @@ func TestGitFetchesTotal_IdleTargetNeverFetches(t *testing.T) { } for _, reason := range []string{ - fetchReasonBootstrap, fetchReasonPublication, fetchReasonRecovery, fetchReasonContention, @@ -260,9 +253,8 @@ func TestGitFetchesTotal_IdleTargetNeverFetches(t *testing.T) { // list docs/interpreting-metrics.md publishes. func TestGitFetchesTotal_ReasonSetMatchesTheDocumentedOne(t *testing.T) { assert.Equal(t, - []string{"bootstrap", "publication", "recovery", "contention", "push_failure_probe", "forced_recheck"}, + []string{"publication", "recovery", "contention", "push_failure_probe", "forced_recheck"}, []string{ - fetchReasonBootstrap, fetchReasonPublication, fetchReasonRecovery, fetchReasonContention, @@ -295,8 +287,7 @@ func TestGitFetchesTotal_ResyncWithRetainedWritesStillReadsTheRemote(t *testing. loop.lastPushAt = time.Now() defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("retained", "alice", "live")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapEvent("retained", "alice", "live")}, }}) require.True(t, loop.finalizeOpenWindow()) require.Len(t, loop.pendingWrites, 1, "the resync must arrive with work retained") @@ -340,8 +331,7 @@ func TestGitFetchesTotal_RecoveryIsOneSeriesWhetherOrNotWritesAreRetained(t *tes // Retained work, so the loop's replay path is the one that recovers. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("retained", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapEvent("retained", "alice", "team-a")}, }}) require.True(t, loop.finalizeOpenWindow()) require.Len(t, loop.pendingWrites, 1) diff --git a/internal/git/branch_worker_metrics_test.go b/internal/git/branch_worker_metrics_test.go index e9c1bff05..02313b25a 100644 --- a/internal/git/branch_worker_metrics_test.go +++ b/internal/git/branch_worker_metrics_test.go @@ -12,6 +12,7 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + k8stypes "k8s.io/apimachinery/pkg/types" "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" "github.com/ConfigButler/gitops-reverser/internal/telemetry" @@ -33,6 +34,7 @@ const testRepoURL = "https://example.com/test" func newMetricsTestWorker() *BranchWorker { return &BranchWorker{ + GitBranch: k8stypes.NamespacedName{Namespace: "test-ns", Name: "test-branch"}, GitProviderRef: "test-provider", GitProviderNamespace: "test-ns", Branch: "main", @@ -46,9 +48,10 @@ func newMetricsTestWorker() *BranchWorker { func workerLabels() map[string]string { return map[string]string{ - "provider_namespace": "test-ns", - "provider_name": "test-provider", - "branch": "main", + "gitbranch_namespace": "test-ns", + "gitbranch_name": "test-branch", + "provider_namespace": "test-ns", + "provider_name": "test-provider", } } @@ -59,10 +62,10 @@ func commitAndPush(w *BranchWorker, writes []PendingWrite, _ int) { w.publishCommitsForPush(writes) } -// commits_total must be labelled with the recording worker's identity -// {provider_namespace, provider_name, branch}. Without labels the counter is a single global +// commits_total must be labelled with the recording worker's GitBranch and its GitProvider +// {gitbranch_namespace, gitbranch_name, provider_namespace, provider_name}. Without labels the counter is a single global // series; with them, concurrent workers (and parallel e2e specs, isolated by their per-suite -// GitProvider namespace) count separately. +// GitBranch namespace) count separately. func TestRecordPendingWritesMetrics_LabelsCommitsByWorkerIdentity(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) @@ -75,10 +78,10 @@ func TestRecordPendingWritesMetrics_LabelsCommitsByWorkerIdentity(t *testing.T) assert.Equal(t, int64(2), commits) } -// A query scoped to a different provider_namespace must not see this worker's +// A query scoped to a different gitbranch_namespace must not see this worker's // commits — this is the property that lets parallel e2e specs each assert on only -// their own commits (each suite mints its GitProvider in a unique namespace). -func TestRecordPendingWritesMetrics_CommitsIsolatedByProviderNamespace(t *testing.T) { +// their own commits (each suite mints its GitBranch in a unique namespace). +func TestRecordPendingWritesMetrics_CommitsIsolatedByGitBranchNamespace(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) @@ -86,9 +89,9 @@ func TestRecordPendingWritesMetrics_CommitsIsolatedByProviderNamespace(t *testin commitAndPush(w, committerWrites(1), 1) otherNamespace := workerLabels() - otherNamespace["provider_namespace"] = "other-suite-ns" + otherNamespace["gitbranch_namespace"] = "other-suite-ns" _, ok := telemetry.CollectInt64Sum(reader, commitsTotalMetric, otherNamespace) - assert.False(t, ok, "a different provider_namespace must not match this worker's commits") + assert.False(t, ok, "a different gitbranch_namespace must not match this worker's commits") } func TestRecordPendingWritesMetrics_LabelsCommitsByAuthorKind(t *testing.T) { @@ -193,8 +196,8 @@ func TestQueueDepthGauge_ReportsWorkTheLoopHasNotReachedYet(t *testing.T) { // Work is accepted. The loop is not running at all, which is the worst case of "has not got to // it yet", and no metric call is made anywhere on this path. - w.enqueueRequest(&WriteRequest{Events: []Event{{}}, CommitMode: CommitModePerEvent}) - w.enqueueRequest(&WriteRequest{Events: []Event{{}}, CommitMode: CommitModePerEvent}) + w.enqueueRequest(&WriteRequest{Events: []Event{{}}}) + w.enqueueRequest(&WriteRequest{Events: []Event{{}}}) depth, ok := telemetry.CollectInt64Sum(reader, queueDepthMetric, workerLabels()) require.True(t, ok, "the scrape must observe the queue even with the loop stopped") @@ -209,8 +212,8 @@ func TestEnqueueRequest_FullQueueCountsTheDrop(t *testing.T) { w := newMetricsTestWorker() w.eventQueue = make(chan WorkItem, 1) - require.True(t, w.enqueueRequest(&WriteRequest{Events: []Event{{}}, CommitMode: CommitModePerEvent})) - require.False(t, w.enqueueRequest(&WriteRequest{Events: []Event{{}}, CommitMode: CommitModePerEvent}), + require.True(t, w.enqueueRequest(&WriteRequest{Events: []Event{{}}})) + require.False(t, w.enqueueRequest(&WriteRequest{Events: []Event{{}}}), "the second request must be refused by the full queue") labels := workerLabels() @@ -227,8 +230,8 @@ func TestEnqueueRequest_FullQueueCountsTheDrop(t *testing.T) { func TestHandleShutdown_DrainsBufferedItemsToZeroDepth(t *testing.T) { w := newMetricsTestWorker() // Two write requests accepted onto the queue but never handled by the loop. - w.enqueueRequest(&WriteRequest{Events: []Event{{}}, CommitMode: CommitModePerEvent}) - w.enqueueRequest(&WriteRequest{Events: []Event{{}}, CommitMode: CommitModePerEvent}) + w.enqueueRequest(&WriteRequest{Events: []Event{{}}}) + w.enqueueRequest(&WriteRequest{Events: []Event{{}}}) require.Equal(t, int64(2), w.inflightItems.Load()) loop := newBranchWorkerEventLoop(w, time.Second) @@ -467,11 +470,10 @@ func TestRecordCommitFailure_SeparatesRefusalFromWriteFault(t *testing.T) { require.NoError(t, err) w := newMetricsTestWorker() - w.recordCommitFailure(commitFailureKindWindow, commitFailureRefused) - w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) + w.recordCommitFailure(commitFailureRefused) + w.recordCommitFailure(commitFailureError) refused := workerLabels() - refused["kind"] = commitFailureKindWindow refused["reason"] = commitFailureRefused count, ok := telemetry.CollectInt64Sum(reader, commitFailuresMetric, refused) require.True(t, ok, "a refused window must be counted") @@ -480,7 +482,6 @@ func TestRecordCommitFailure_SeparatesRefusalFromWriteFault(t *testing.T) { // The two reasons need different people: a refusal is a Git path a human has to fix and will // not clear on its own, while an error may be transient. Folding them would hide that. faulted := workerLabels() - faulted["kind"] = commitFailureKindAtomic faulted["reason"] = commitFailureError count, ok = telemetry.CollectInt64Sum(reader, commitFailuresMetric, faulted) require.True(t, ok) diff --git a/internal/git/branch_worker_new_branch_test.go b/internal/git/branch_worker_new_branch_test.go index 37993656b..1fe970f00 100644 --- a/internal/git/branch_worker_new_branch_test.go +++ b/internal/git/branch_worker_new_branch_test.go @@ -170,8 +170,7 @@ func (f *newBranchFixture) localHead() (plumbing.ReferenceName, plumbing.Hash) { // liveWrite sends one live ConfigMap event through the loop's commit window. func liveWrite(loop *branchWorkerEventLoop, name string) { loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent(name, "alice", newBranchTarget)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent(name, "alice", newBranchTarget)}, }}) } diff --git a/internal/git/branch_worker_parent_change_test.go b/internal/git/branch_worker_parent_change_test.go index 3518ff7d1..2d1576c75 100644 --- a/internal/git/branch_worker_parent_change_test.go +++ b/internal/git/branch_worker_parent_change_test.go @@ -25,7 +25,7 @@ import ( // allFetchReasons is every series a fetch can be counted under. var allFetchReasons = []string{ - fetchReasonBootstrap, fetchReasonPublication, fetchReasonRecovery, fetchReasonContention, + fetchReasonPublication, fetchReasonRecovery, fetchReasonContention, fetchReasonPushFailureProbe, fetchReasonRefresh, } diff --git a/internal/git/branch_worker_split_test.go b/internal/git/branch_worker_split_test.go index 45061b9fb..d6e7139da 100644 --- a/internal/git/branch_worker_split_test.go +++ b/internal/git/branch_worker_split_test.go @@ -718,60 +718,6 @@ func TestBranchWorker_PushFollowedByFetchFailure_TreatsAsTransient(t *testing.T) assert.Equal(t, remoteRefBefore.Hash(), remoteRefAfter.Hash()) } -func TestBranchWorker_AtomicAndGroupedInterleaved_PreservesArrivalOrder(t *testing.T) { - worker, serverRepo, _ := setupCommitPushSplitWorker(t) - - groupedFirst, err := worker.buildGroupedPendingWrite( - worker.ctx, - []Event{configMapEvent("grouped-first", "alice", "team-a")}, - ) - require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*groupedFirst})) - - atomicRequest := &WriteRequest{ - Events: []Event{configMapEvent("atomic-second", "reconciler", "team-a")}, - CommitMode: CommitModeAtomic, - CommitMessage: "atomic: second write", - } - atomicSecond, err := worker.buildAtomicPendingWrite(worker.ctx, atomicRequest) - require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*atomicSecond})) - - groupedThird, err := worker.buildGroupedPendingWrite( - worker.ctx, - []Event{configMapEvent("grouped-third", "bob", "team-a")}, - ) - require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*groupedThird})) - - require.NoError(t, worker.pushPendingCommits([]PendingWrite{*groupedFirst, *atomicSecond, *groupedThird})) - - headRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.NoError(t, err) - headCommit, err := serverRepo.CommitObject(headRef.Hash()) - require.NoError(t, err) - require.Len(t, headCommit.ParentHashes, 1) - assert.Equal( - t, - expectSingleLiveMessage("CREATE", "v1", "configmaps", "default", "grouped-third"), - headCommit.Message, - ) - - atomicCommit, err := serverRepo.CommitObject(headCommit.ParentHashes[0]) - require.NoError(t, err) - require.Len(t, atomicCommit.ParentHashes, 1) - assert.Equal(t, "atomic: second write", atomicCommit.Message) - - firstCommit, err := serverRepo.CommitObject(atomicCommit.ParentHashes[0]) - require.NoError(t, err) - require.Len(t, firstCommit.ParentHashes, 1) - assert.Equal( - t, - expectSingleLiveMessage("CREATE", "v1", "configmaps", "default", "grouped-first"), - firstCommit.Message, - ) -} - func TestBranchWorker_Replay_DropsUnitsThatBecomeNoOpAgainstNewRemoteTree(t *testing.T) { worker, serverRepo, remoteURL := setupCommitPushSplitWorker(t) @@ -830,7 +776,6 @@ func TestEventLoop_CommitWindowZero_HonestPerEvent(t *testing.T) { configMapEvent("first", "alice", "team-a"), configMapEvent("second", "alice", "team-a"), }, - CommitMode: CommitModePerEvent, }}) assert.Nil(t, loop.openWindow, "commitWindow=0 must finalize the open window on every event") @@ -867,7 +812,6 @@ func TestEventLoop_AuthorChangeFinalizesOpenWindow(t *testing.T) { configMapEvent("first", "alice", "team-a"), configMapEvent("second", "bob", "team-a"), }, - CommitMode: CommitModePerEvent, }}) require.Len(t, loop.pendingWrites, 1) @@ -891,7 +835,6 @@ func TestEventLoop_TargetChangeFinalizesOpenWindow(t *testing.T) { configMapTargetEvent("first", "alice", "team-a"), configMapTargetEvent("second", "alice", "team-b"), }, - CommitMode: CommitModePerEvent, }}) require.Len(t, loop.pendingWrites, 1) @@ -913,8 +856,7 @@ func TestEventLoop_RepeatedPathEditsCollapseInOpenWindow(t *testing.T) { second.Object.Object["data"] = map[string]interface{}{"key": "latest"} loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{first, second, configMapEvent("other", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{first, second, configMapEvent("other", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) @@ -931,32 +873,6 @@ func TestEventLoop_RepeatedPathEditsCollapseInOpenWindow(t *testing.T) { loop.stopTimers() } -func TestEventLoop_AtomicRequestFinalizesOpenWindowFirst(t *testing.T) { - worker, _, _ := setupCommitPushSplitWorker(t) - loop := newBranchWorkerEventLoop(worker, time.Hour) - loop.lastPushAt = time.Now() - - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("live", "alice", "team-a")}, - CommitMode: CommitModePerEvent, - }}) - require.NotNil(t, loop.openWindow) - - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("atomic", "reconciler", "team-a")}, - CommitMode: CommitModeAtomic, - }}) - - assert.Nil(t, loop.openWindow) - require.Len(t, loop.pendingWrites, 2) - assert.Equal(t, PendingWriteCommit, loop.pendingWrites[0].Kind) - assert.Equal(t, "live", loop.pendingWrites[0].Events[0].Identifier.Name) - assert.Equal(t, PendingWriteAtomic, loop.pendingWrites[1].Kind) - assert.Equal(t, "atomic", loop.pendingWrites[1].Events[0].Identifier.Name) - - loop.stopTimers() -} - func TestEventLoop_ByteCapFinalizesIncludingTrippingEvent(t *testing.T) { worker, _, _ := setupCommitPushSplitWorker(t) worker.branchBufferMaxBytes = 1 @@ -964,8 +880,7 @@ func TestEventLoop_ByteCapFinalizesIncludingTrippingEvent(t *testing.T) { loop.lastPushAt = time.Now() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("oversized", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapEvent("oversized", "alice", "team-a")}, }}) assert.Nil(t, loop.openWindow) @@ -982,15 +897,13 @@ func TestEventLoop_AuthorChangeStartsFreshCommitTimer(t *testing.T) { loop.lastPushAt = time.Now() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("first", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapEvent("first", "alice", "team-a")}, }}) require.NotNil(t, loop.commitTimer) firstTimer := loop.commitTimer loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("second", "bob", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapEvent("second", "bob", "team-a")}, }}) require.NotNil(t, loop.commitTimer) @@ -1016,8 +929,7 @@ func TestEventLoop_CrossAuthorSamePathPingPongUsesOnlyAuthorBoundaries(t *testin // changes nothing settles at once, and would leave no window to count. event.Object.Object["data"] = map[string]interface{}{"key": fmt.Sprintf("v%d", i)} loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{event}, - CommitMode: CommitModePerEvent, + Events: []Event{event}, }}) } loop.finalizeOpenWindow() @@ -1037,42 +949,7 @@ func TestEventLoop_CrossAuthorSamePathPingPongUsesOnlyAuthorBoundaries(t *testin loop.stopTimers() } -func TestEventLoop_AtomicRequest_RespectsCooldownAndUsesNormalPushPath(t *testing.T) { - worker, serverRepo, _ := setupCommitPushSplitWorker(t) - - initialRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.NoError(t, err) - - loop := newBranchWorkerEventLoop(worker, time.Second) - loop.lastPushAt = time.Now() - - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("atomic", "reconciler", "team-a")}, - CommitMode: CommitModeAtomic, - }}) - - assert.Nil(t, loop.openWindow, "atomic requests should not remain in the live-event window") - assert.Len(t, loop.pendingWrites, 1, - "atomic requests should join pendingWrites and wait for the normal push cycle") - assert.NotNil(t, loop.pushTimer, - "active cooldown should defer the push with the regular timer path") - - afterCommitRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.NoError(t, err) - assert.Equal(t, initialRef.Hash(), afterCommitRef.Hash(), - "remote must not advance while cooldown defers the push") - - localRepo, err := git.PlainOpen(worker.repoPath()) - require.NoError(t, err) - localRef, err := localRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.NoError(t, err) - assert.NotEqual(t, initialRef.Hash(), localRef.Hash(), - "local branch should already contain the atomic commit") - - loop.stopTimers() -} - -func TestEventLoop_DeferredEventCommitsAndAtomicDuringCooldownPushTogether(t *testing.T) { +func TestEventLoop_DeferredEventCommitsDuringCooldownPushTogether(t *testing.T) { worker, serverRepo, _ := setupCommitPushSplitWorker(t) createPlainGitTarget(t, worker, "target-a", "iso-a") createPlainGitTarget(t, worker, "target-b", "iso-b") @@ -1085,61 +962,46 @@ func TestEventLoop_DeferredEventCommitsAndAtomicDuringCooldownPushTogether(t *te loop.lastPushAt = time.Now() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("live-a", "alice", "target-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("live-a", "alice", "target-a")}, }}) loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("live-b", "bob", "target-b")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("live-b", "bob", "target-b")}, }}) require.Len(t, loop.pendingWrites, 2, "deferred event commits during cooldown are retained as local commits, not lost") loop.publishLoopState(0) - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapEvent("snapshot-only", "reconciler", "")}, - CommitMode: CommitModeAtomic, - GitTargetName: "target-b", - GitTargetNamespace: "default", - }}) - require.Len(t, loop.pendingWrites, 3) - loop.pushPending() finalRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) require.NoError(t, err) commits := commitsAfterHash(t, serverRepo, finalRef.Hash(), initialHash) - require.Len(t, commits, 3) + require.Len(t, commits, 2) assert.Equal(t, expectSingleLiveMessage("CREATE", "v1", "configmaps", "default", "live-a"), commits[0].Message) assert.Equal(t, expectSingleLiveMessage("CREATE", "v1", "configmaps", "default", "live-b"), commits[1].Message) - assert.Equal(t, "chore: reconcile 1 resources", commits[2].Message) loop.stopTimers() } -func TestEventLoop_AtomicPushFailure_DoesNotAdvanceCooldownOrLosePendingWrite(t *testing.T) { +func TestEventLoop_PushFailure_DoesNotAdvanceCooldownOrLosePendingWrite(t *testing.T) { worker, _, remoteURL := setupCommitPushSplitWorker(t) - request := &WriteRequest{ - Events: []Event{configMapEvent("atomic", "reconciler", "team-a")}, - CommitMode: CommitModeAtomic, - } - - pendingWrite, err := worker.buildAtomicPendingWrite(worker.ctx, request) + pendingWrite, err := worker.buildGroupedPendingWrite( + worker.ctx, []Event{configMapEvent("live", "alice", "team-a")}) require.NoError(t, err) loop := newBranchWorkerEventLoop(worker, time.Second) loop.decide(*pendingWrite) loop.commitDecided() - require.Len(t, loop.pendingWrites, 1, "the atomic write is committed and pending its push") + require.Len(t, loop.pendingWrites, 1, "the write is committed and pending its push") remotePath := strings.TrimPrefix(remoteURL, "file://") require.NoError(t, os.RemoveAll(remotePath)) loop.pushPending() - assert.True(t, loop.lastPushAt.IsZero(), "failed atomic push must not advance cooldown state") - assert.Len(t, loop.pendingWrites, 1, "failed atomic push must retain pending work for retry") + assert.True(t, loop.lastPushAt.IsZero(), "a failed push must not advance cooldown state") + assert.Len(t, loop.pendingWrites, 1, "a failed push must retain pending work for retry") } // TestResync_WorkerAppliesMarkAndSweepAndCommits drives a resync through the worker @@ -1160,8 +1022,7 @@ func TestResync_WorkerAppliesMarkAndSweepAndCommits(t *testing.T) { // Seed a managed ConfigMap under the GitTarget path with a normal per-event commit. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("drop-me", "alice", "target-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("drop-me", "alice", "target-a")}, }}) require.Len(t, loop.pendingWrites, 1) @@ -1232,8 +1093,7 @@ func TestResync_WorkerEmptyDesiredSweepsManagedResource(t *testing.T) { loop := newBranchWorkerEventLoop(worker, 0) loop.lastPushAt = time.Now() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("orphan", "alice", "target-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("orphan", "alice", "target-a")}, }}) require.Len(t, loop.pendingWrites, 1) diff --git a/internal/git/branch_worker_test.go b/internal/git/branch_worker_test.go index c8cc7a016..1883075f4 100644 --- a/internal/git/branch_worker_test.go +++ b/internal/git/branch_worker_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "filippo.io/age" "github.com/fluxcd/pkg/apis/meta" "github.com/go-git/go-git/v6" "github.com/go-git/go-git/v6/config" @@ -23,7 +22,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" @@ -114,444 +112,6 @@ func TestBranchWorker_IdentityFields(t *testing.T) { } } -func TestBranchWorker_EnsurePathBootstrapped_EmptyPathCreatesTemplate(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - serverRepo := createBareRepo(t, remotePath) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{ - URL: remoteURL, - }, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - createTargetWithEncryption(ctx, t, k8sClient, "bootstrap-target", "default", "test-repo", "main", "clusters/prod") - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/prod", "bootstrap-target", "default")) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/prod", "bootstrap-target", "default")) - - _, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "Bootstrap should not create the branch remotely") - - repoPath := worker.repoPath() - _, err = os.Stat(filepath.Join(repoPath, "clusters/prod", "README.md")) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(repoPath, "clusters/prod", sopsConfigFileName)) - require.NoError(t, err) - // The fully-commented .gittargetignore escape hatch is staged alongside the other - // bootstrap artifacts and must self-accept (it ignores nothing until edited). - _, err = os.Stat(filepath.Join(repoPath, "clusters/prod", gitTargetIgnoreFileName)) - require.NoError(t, err) -} - -func TestBranchWorker_EnsurePathBootstrapped_NonEmptyPathBootstrapsMissingFiles(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - serverRepo := createBareRepo(t, remotePath) - seedPath := filepath.Join(tempDir, "seed") - seedRepo, seedWorktree := initLocalRepo(t, seedPath, remoteURL, "main") - require.NoError(t, os.MkdirAll(filepath.Join(seedPath, "clusters/prod"), 0750)) - commitFileChange(t, seedWorktree, seedPath, "clusters/prod/existing.txt", "already populated") - require.NoError(t, seedRepo.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{ - config.RefSpec("refs/heads/main:refs/heads/main"), - }, - })) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{ - URL: remoteURL, - }, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - createTargetWithEncryption(ctx, t, k8sClient, "bootstrap-target", "default", "test-repo", "main", "clusters/prod") - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/prod", "bootstrap-target", "default")) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/prod", "bootstrap-target", "default")) - - ref, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.NoError(t, err) - assert.Equal(t, 1, countDepth(t, serverRepo, ref.Hash()), "Bootstrap should not create a remote commit") - - repoPath := worker.repoPath() - _, err = os.Stat(filepath.Join(repoPath, "clusters/prod", "README.md")) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(repoPath, "clusters/prod", sopsConfigFileName)) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(repoPath, "clusters/prod", "existing.txt")) - require.NoError(t, err) -} - -func TestBranchWorker_EnsurePathBootstrapped_NoEncryptionSkipsSOPSConfig(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - _ = createBareRepo(t, remotePath) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{ - URL: remoteURL, - }, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - createTargetWithoutEncryption(ctx, t, k8sClient, "bootstrap-target", "default", "test-repo", "main", "clusters/dev") - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/dev", "bootstrap-target", "default")) - - repoPath := worker.repoPath() - _, err := os.Stat(filepath.Join(repoPath, "clusters/dev", "README.md")) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(repoPath, "clusters/dev", sopsConfigFileName)) - assert.True(t, os.IsNotExist(err), "Bootstrap SOPS config should be skipped when encryption is not configured") -} - -func TestBranchWorker_EnsurePathBootstrapped_ExistingFileNotOverwritten(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - _ = createBareRepo(t, remotePath) - - seedPath := filepath.Join(tempDir, "seed") - seedRepo, seedWorktree := initLocalRepo(t, seedPath, remoteURL, "main") - require.NoError(t, os.MkdirAll(filepath.Join(seedPath, "clusters/prod"), 0750)) - customREADME := "# custom readme\n" - commitFileChange(t, seedWorktree, seedPath, "clusters/prod/README.md", customREADME) - require.NoError(t, seedRepo.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{ - config.RefSpec("refs/heads/main:refs/heads/main"), - }, - })) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{ - URL: remoteURL, - }, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - createTargetWithEncryption(ctx, t, k8sClient, "bootstrap-target", "default", "test-repo", "main", "clusters/prod") - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/prod", "bootstrap-target", "default")) - - repoPath := worker.repoPath() - readmeContent, err := os.ReadFile(filepath.Join(repoPath, "clusters/prod", "README.md")) - require.NoError(t, err) - assert.Equal(t, customREADME, string(readmeContent), "Bootstrap must not overwrite existing files") - _, err = os.Stat(filepath.Join(repoPath, "clusters/prod", sopsConfigFileName)) - require.NoError(t, err) -} - -func TestBranchWorker_EnsurePathBootstrapped_EnableEncryptionLaterAddsSOPSConfig(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - serverRepo := createBareRepo(t, remotePath) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{ - URL: remoteURL, - }, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - createTargetWithoutEncryption(ctx, t, k8sClient, "bootstrap-target", "default", "test-repo", "main", "clusters/dev") - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/dev", "bootstrap-target", "default")) - - repoPath := worker.repoPath() - _, err := os.Stat(filepath.Join(repoPath, "clusters/dev", "README.md")) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(repoPath, "clusters/dev", sopsConfigFileName)) - assert.True(t, os.IsNotExist(err), "SOPS config should not exist before encryption is configured") - - attachEncryptionToTarget(ctx, t, k8sClient, "bootstrap-target", "default") - require.NoError(t, worker.EnsurePathBootstrapped("clusters/dev", "bootstrap-target", "default")) - - _, err = serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "Bootstrap should not create a remote commit") - _, err = os.Stat(filepath.Join(repoPath, "clusters/dev", "README.md")) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(repoPath, "clusters/dev", sopsConfigFileName)) - require.NoError(t, err) -} - -func TestBranchWorker_EnsurePathBootstrapped_InvalidEncryptionSecretSkipsSOPSConfig(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - _ = createBareRepo(t, remotePath) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{ - URL: remoteURL, - }, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - - createTargetWithEncryptionSecretData( - ctx, - t, - k8sClient, - "bootstrap-target", - "default", - "test-repo", - "main", - "clusters/dev", - map[string][]byte{ - "identity.agekey": []byte("not-an-age-identity"), - }, - ) - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/dev", "bootstrap-target", "default")) - - repoPath := worker.repoPath() - _, err := os.Stat(filepath.Join(repoPath, "clusters/dev", "README.md")) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(repoPath, "clusters/dev", sopsConfigFileName)) - assert.True(t, os.IsNotExist(err), "Bootstrap SOPS config should be skipped for invalid encryption secret") -} - -func TestBranchWorker_EnsurePathBootstrapped_MissingSOPSKeySkipsSOPSConfig(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - _ = createBareRepo(t, remotePath) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{ - URL: remoteURL, - }, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - - createTargetWithEncryptionSecretData( - ctx, - t, - k8sClient, - "bootstrap-target", - "default", - "test-repo", - "main", - "clusters/dev", - map[string][]byte{ - "OTHER_ENV": []byte("value"), - }, - ) - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/dev", "bootstrap-target", "default")) - - repoPath := worker.repoPath() - _, err := os.Stat(filepath.Join(repoPath, "clusters/dev", "README.md")) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(repoPath, "clusters/dev", sopsConfigFileName)) - assert.True(t, os.IsNotExist(err), "Bootstrap SOPS config should be skipped when no .agekey entry is present") -} - -func TestBranchWorker_EnsurePathBootstrapped_RendersAllResolvedRecipients(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - _ = createBareRepo(t, remotePath) - - secretIdentity, err := age.GenerateX25519Identity() - require.NoError(t, err) - secondaryIdentity, err := age.GenerateX25519Identity() - require.NoError(t, err) - publicOnlyIdentity, err := age.GenerateX25519Identity() - require.NoError(t, err) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{ - URL: remoteURL, - }, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - - encryptionSecret := &corev1.Secret{} - encryptionSecret.Name = "sops-age-key" - encryptionSecret.Namespace = "default" - encryptionSecret.Data = map[string][]byte{ - "identity.agekey": []byte(secretIdentity.String()), - "backup.agekey": []byte(secondaryIdentity.String()), - } - require.NoError(t, k8sClient.Create(ctx, encryptionSecret)) - - target := &configv1alpha3.GitTarget{} - target.Name = "bootstrap-target" - target.Namespace = "default" - target.Spec.GitBranchRef = meta.NamespacedObjectReference{Name: testGitBranchName("test-repo", "main")} - target.Spec.Path = "clusters/dev" - target.Spec.Encryption = &configv1alpha3.EncryptionSpec{ - Provider: "sops", - SecretRef: meta.LocalObjectReference{ - Name: encryptionSecret.Name, - }, - Age: &configv1alpha3.AgeEncryptionSpec{ - Enabled: true, - Recipients: configv1alpha3.AgeRecipientsSpec{ - PublicKeys: []string{ - publicOnlyIdentity.Recipient().String(), - }, - ExtractFromSecret: true, - }, - }, - } - require.NoError(t, k8sClient.Create(ctx, target)) - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - require.NoError(t, worker.EnsurePathBootstrapped("clusters/dev", "bootstrap-target", "default")) - - repoPath := worker.repoPath() - sopsConfig, err := os.ReadFile(filepath.Join(repoPath, "clusters/dev", sopsConfigFileName)) - require.NoError(t, err) - assert.Contains(t, string(sopsConfig), secretIdentity.Recipient().String()) - assert.Contains(t, string(sopsConfig), secondaryIdentity.Recipient().String()) - assert.Contains(t, string(sopsConfig), publicOnlyIdentity.Recipient().String()) -} - func TestBranchWorker_CommitAndPushRequest_PreparesRepositoryBeforeFirstWrite(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() @@ -897,127 +457,6 @@ func TestBranchWorker_CommitAndPushRequest_UsesProviderCommitterAndTargetMessage assert.Equal(t, ConstructSafeEmail("alice", "cluster.local"), commit.Author.Email) } -func TestBranchWorker_CommitAndPushRequest_UsesBatchTemplateForAtomicRequest(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - remotePath := filepath.Join(tempDir, "remote.git") - remoteURL := "file://" + remotePath - serverRepo := createBareRepo(t, remotePath) - - seedPath := filepath.Join(tempDir, "seed") - seedRepo, seedWorktree := initLocalRepo(t, seedPath, remoteURL, "main") - commitFileChange(t, seedWorktree, seedPath, "README.md", "seed\n") - require.NoError(t, seedRepo.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{ - config.RefSpec("refs/heads/main:refs/heads/main"), - }, - })) - - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = configv1alpha3.AddToScheme(scheme) - k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - provider := &configv1alpha3.GitProvider{ - Spec: configv1alpha3.GitProviderSpec{URL: remoteURL}, - } - provider.Name = "test-repo" - provider.Namespace = "default" - require.NoError(t, k8sClient.Create(ctx, provider)) - - target := &configv1alpha3.GitTarget{} - target.Name = "demo-target" - target.Namespace = "default" - target.Spec.GitBranchRef = meta.NamespacedObjectReference{Name: testGitBranchName("test-repo", "main")} - target.Spec.Path = "clusters/dev" - target.Spec.Commit = &configv1alpha3.GitTargetCommitSpec{ - Message: &configv1alpha3.CommitMessageSpec{ - ReconcileTemplate: "reconcile({{.GitTarget}}): {{.Count}} resources", - }, - } - require.NoError(t, k8sClient.Create(ctx, target)) - - worker := NewBranchWorker( - k8sClient, - logr.Discard(), - "test-repo", - "default", - "main", - RepoIdentity{URL: remoteURL}, - nil, - BranchWorkerLimits{}, - ) - worker.ctx = ctx - - request := &WriteRequest{ - Events: []Event{ - { - Operation: "CREATE", - Identifier: itypes.ResourceIdentifier{ - Group: "", - Version: "v1", - Resource: "configmaps", - Namespace: "default", - Name: "first", - }, - Object: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": map[string]interface{}{ - "name": "first", - "namespace": "default", - }, - }, - }, - UserInfo: UserInfo{Username: "ignored-atomic-author"}, - Path: "clusters/dev", - }, - { - Operation: "CREATE", - Identifier: itypes.ResourceIdentifier{ - Group: "", - Version: "v1", - Resource: "configmaps", - Namespace: "default", - Name: "second", - }, - Object: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": map[string]interface{}{ - "name": "second", - "namespace": "default", - }, - }, - }, - UserInfo: UserInfo{Username: "ignored-atomic-author"}, - Path: "clusters/dev", - }, - }, - CommitMode: CommitModeAtomic, - GitTargetName: "demo-target", - GitTargetNamespace: "default", - } - - pendingWrite, err := worker.buildAtomicPendingWrite(worker.ctx, request) - require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) - require.NoError(t, worker.pushPendingCommits([]PendingWrite{*pendingWrite})) - - remoteHeadRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.NoError(t, err) - - commit, err := serverRepo.CommitObject(remoteHeadRef.Hash()) - require.NoError(t, err) - assert.Equal(t, "reconcile(demo-target): 2 resources", commit.Message) - assert.Equal(t, DefaultCommitterName, commit.Committer.Name) - assert.Equal(t, DefaultCommitterEmail, commit.Committer.Email) - assert.Equal(t, DefaultCommitterName, commit.Author.Name) - assert.Equal(t, DefaultCommitterEmail, commit.Author.Email) -} - func TestBranchWorker_CommitAndPushRequest_SignsCommitWhenConfigured(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() @@ -1221,126 +660,3 @@ func TestBranchWorker_CommitAndPushRequest_SkipsWriteWhenSigningSecretIsInvalid( require.NoError(t, err) assert.Equal(t, initialRef.Hash(), finalRef.Hash()) } - -func createTargetWithEncryption( - ctx context.Context, - t *testing.T, - k8sClient client.Client, - name, namespace, providerName, branch, path string, -) { - t.Helper() - identity, err := age.GenerateX25519Identity() - require.NoError(t, err) - - encryptionSecret := &corev1.Secret{} - encryptionSecret.Name = "sops-age-key" - encryptionSecret.Namespace = namespace - encryptionSecret.Data = map[string][]byte{ - "identity.agekey": []byte(identity.String()), - } - require.NoError(t, k8sClient.Create(ctx, encryptionSecret)) - - target := &configv1alpha3.GitTarget{} - target.Name = name - target.Namespace = namespace - target.Spec.GitBranchRef = meta.NamespacedObjectReference{Name: testGitBranchName(providerName, branch)} - target.Spec.Path = path - target.Spec.Encryption = &configv1alpha3.EncryptionSpec{ - Provider: "sops", - SecretRef: meta.LocalObjectReference{ - Name: "sops-age-key", - }, - Age: &configv1alpha3.AgeEncryptionSpec{ - Enabled: true, - Recipients: configv1alpha3.AgeRecipientsSpec{ - ExtractFromSecret: true, - }, - }, - } - require.NoError(t, k8sClient.Create(ctx, target)) -} - -func createTargetWithoutEncryption( - ctx context.Context, - t *testing.T, - k8sClient client.Client, - name, namespace, providerName, branch, path string, -) { - t.Helper() - target := &configv1alpha3.GitTarget{} - target.Name = name - target.Namespace = namespace - target.Spec.GitBranchRef = meta.NamespacedObjectReference{Name: testGitBranchName(providerName, branch)} - target.Spec.Path = path - require.NoError(t, k8sClient.Create(ctx, target)) -} - -func createTargetWithEncryptionSecretData( - ctx context.Context, - t *testing.T, - k8sClient client.Client, - name, namespace, providerName, branch, path string, - secretData map[string][]byte, -) { - t.Helper() - - encryptionSecret := &corev1.Secret{} - encryptionSecret.Name = "sops-age-key" - encryptionSecret.Namespace = namespace - encryptionSecret.Data = secretData - require.NoError(t, k8sClient.Create(ctx, encryptionSecret)) - - target := &configv1alpha3.GitTarget{} - target.Name = name - target.Namespace = namespace - target.Spec.GitBranchRef = meta.NamespacedObjectReference{Name: testGitBranchName(providerName, branch)} - target.Spec.Path = path - target.Spec.Encryption = &configv1alpha3.EncryptionSpec{ - Provider: "sops", - SecretRef: meta.LocalObjectReference{ - Name: encryptionSecret.Name, - }, - Age: &configv1alpha3.AgeEncryptionSpec{ - Enabled: true, - Recipients: configv1alpha3.AgeRecipientsSpec{ - ExtractFromSecret: true, - }, - }, - } - require.NoError(t, k8sClient.Create(ctx, target)) -} - -func attachEncryptionToTarget( - ctx context.Context, - t *testing.T, - k8sClient client.Client, - targetName, targetNamespace string, -) { - t.Helper() - identity, err := age.GenerateX25519Identity() - require.NoError(t, err) - - encryptionSecret := &corev1.Secret{} - encryptionSecret.Name = "sops-age-key" - encryptionSecret.Namespace = targetNamespace - encryptionSecret.Data = map[string][]byte{ - "identity.agekey": []byte(identity.String()), - } - require.NoError(t, k8sClient.Create(ctx, encryptionSecret)) - - target := &configv1alpha3.GitTarget{} - require.NoError(t, k8sClient.Get(ctx, client.ObjectKey{Name: targetName, Namespace: targetNamespace}, target)) - target.Spec.Encryption = &configv1alpha3.EncryptionSpec{ - Provider: "sops", - SecretRef: meta.LocalObjectReference{ - Name: encryptionSecret.Name, - }, - Age: &configv1alpha3.AgeEncryptionSpec{ - Enabled: true, - Recipients: configv1alpha3.AgeRecipientsSpec{ - ExtractFromSecret: true, - }, - }, - } - require.NoError(t, k8sClient.Update(ctx, target)) -} diff --git a/internal/git/commit.go b/internal/git/commit.go index bb3c8bd63..d2b82b7ef 100644 --- a/internal/git/commit.go +++ b/internal/git/commit.go @@ -22,25 +22,6 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/types" ) -// renderReconcileCommitMessageFromEvents renders the reconcile commit message for the -// events-based atomic path from the provider's ReconcileTemplate. It carries no single -// type and no snapshot version, so those template fields stay empty (the default guards them). -// Literal overrides are resolved by the caller. -func renderReconcileCommitMessageFromEvents( - events []Event, - gitTarget string, - config CommitConfig, -) (string, error) { - return renderCommitTemplate( - "reconcile", - config.Message.ReconcileTemplate, - ReconcileCommitMessageData{ - Count: len(events), - GitTarget: gitTarget, - }, - ) -} - // renderReconcileCommitMessage renders the reconcile commit message for a resync from the // provider's ReconcileTemplate, so a resync honours a custom reconcile template. count is the // number of resources the reconcile changed; scopeGVR names the synced type for a per-type @@ -123,11 +104,9 @@ func ValidateCommitConfig(config CommitConfig) error { GitTargetName: "example-target", } - if _, err := renderReconcileCommitMessageFromEvents( - []Event{sampleEvent}, - "example-target", - config, - ); err != nil { + // The whole-GitTarget reconcile carries no type and no snapshot version, so those template + // fields render empty; the default template guards them. + if _, err := renderReconcileCommitMessage(1, "example-target", nil, "", config); err != nil { return err } diff --git a/internal/git/commit_executor.go b/internal/git/commit_executor.go index 8cd744655..f892c2075 100644 --- a/internal/git/commit_executor.go +++ b/internal/git/commit_executor.go @@ -215,8 +215,6 @@ const ( // was committed instead. Never returned by resolveMessage: it is decided at render time and // stamped onto the write, because a silent, SUCCESSFUL commit is otherwise invisible. messageResolutionRequestFallback - // messageResolutionReconcileTemplate renders reconcileTemplate from the write's events. - messageResolutionReconcileTemplate // messageResolutionLiveTemplate renders liveTemplate for a live window. messageResolutionLiveTemplate ) @@ -229,17 +227,13 @@ func (p PendingWrite) resolveMessage() messageResolution { return messageResolutionPreRendered // Framing is bound to the GROUPED kind, not merely to "carries a message". A CommitRequest // attaches its message to an open live window, which finalizes as PendingWriteCommit, so that - // is the only write requestTemplate describes. An atomic snapshot can also carry a message - // (WriteRequest.CommitMessage), and framing one with the live context would silently reroute - // the documented atomic path through a template written for save requests. + // is the only write requestTemplate describes. case p.Kind == PendingWriteCommit && p.CommitMessage != "" && p.CommitConfig.Message.RequestTemplate != "": return messageResolutionRequestTemplate case p.CommitMessage != "": return messageResolutionRequest - case p.Kind == PendingWriteAtomic: - return messageResolutionReconcileTemplate case p.Kind == PendingWriteCommit: return messageResolutionLiveTemplate default: @@ -247,9 +241,7 @@ func (p PendingWrite) resolveMessage() messageResolution { } } -// label is the commits_total `message_source` value for this resolution. A resync and an atomic -// snapshot are both reported as reconcile: they differ in how the text is produced, not in where -// an operator would say the message came from. +// label is the commits_total `message_source` value for this resolution. func (r messageResolution) label() string { switch r { case messageResolutionRequest: @@ -258,7 +250,7 @@ func (r messageResolution) label() string { return messageSourceCommitRequestFramed case messageResolutionRequestFallback: return messageSourceCommitRequestFallback - case messageResolutionPreRendered, messageResolutionReconcileTemplate: + case messageResolutionPreRendered: return messageSourceReconcile case messageResolutionLiveTemplate: return messageSourceLive @@ -307,8 +299,6 @@ func (p PendingWrite) commitMetadata() (string, *gogit.CommitOptions, messageRes if err = ValidateLiteralCommitMessage(p.CommitMessage); err == nil { message, resolution = p.framedRequestMessage() } - case messageResolutionReconcileTemplate: - message, err = renderReconcileCommitMessageFromEvents(p.Events, p.Target().Name, p.CommitConfig) case messageResolutionLiveTemplate: message, err = renderLiveCommitMessage(p, p.CommitConfig) case messageResolutionRequestFallback: @@ -369,7 +359,7 @@ func (w *BranchWorker) executePendingWrite( // report ZeroHash to keep the per-write SHA bookkeeping uniform. created, err := w.executeResyncPendingWrite(ctx, repo, worktree, pendingWrite) return created, plumbing.ZeroHash, messageResolutionPreRendered, err - case PendingWriteCommit, PendingWriteAtomic: + case PendingWriteCommit: return w.executeEventWrite(ctx, repo, worktree, pendingWrite) default: return 0, plumbing.ZeroHash, messageResolutionUnsupported, @@ -377,7 +367,7 @@ func (w *BranchWorker) executePendingWrite( } } -// executeEventWrite applies a grouped window's or an atomic batch's events and commits them. The +// executeEventWrite applies a grouped window's events and commits them. The // first time, the write keeps only the events whose plan changed the tree: an object that changed // nothing is no action, and a replay must not plan it again onto a newer tip, where it would revert // what that tip changed (docs/design/applier-echoes.md, F2). A save keeps every object. diff --git a/internal/git/commit_executor_test.go b/internal/git/commit_executor_test.go index e5730bb86..927544de7 100644 --- a/internal/git/commit_executor_test.go +++ b/internal/git/commit_executor_test.go @@ -65,7 +65,7 @@ func newExecutorSecretEvent(path string) Event { func seedEventContent(t *testing.T, repoPath string, worktree *git.Worktree, writer *contentWriter, event Event) { t.Helper() - content, err := writer.buildContentForWrite(context.Background(), event) + content, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) gitPath := generateFilePath(event.Identifier, types.SensitiveResourcePolicy{}) @@ -105,7 +105,7 @@ func TestExecutor_GroupedSingleEvent_UsesLiveTemplate(t *testing.T) { pendingWrite := PendingWrite{ Kind: PendingWriteCommit, - Events: []Event{makeEvent("alice", "api")}, + Events: []Event{makeEvent("api")}, CommitConfig: config, } @@ -123,8 +123,8 @@ func TestExecutor_GroupedMultiEvent_UsesLiveTemplate(t *testing.T) { pendingWrite := PendingWrite{ Kind: PendingWriteCommit, Events: []Event{ - makeEvent("alice", "api"), - makeEvent("alice", "worker"), + makeEvent("api"), + makeEvent("worker"), }, CommitConfig: config, } @@ -136,29 +136,6 @@ func TestExecutor_GroupedMultiEvent_UsesLiveTemplate(t *testing.T) { assert.Equal(t, DefaultCommitterName, options.Committer.Name) } -func TestExecutor_AtomicUnit_UsesReconcileMessage(t *testing.T) { - config := ResolveCommitConfig(nil) - config.Message.ReconcileTemplate = "reconcile: {{.Count}} {{.GitTarget}}" - - pendingWrite := PendingWrite{ - Kind: PendingWriteAtomic, - CommitMessage: "", - Events: []Event{ - makeEvent("alice", "api"), - makeEvent("bob", "worker"), - }, - CommitConfig: config, - GitTargetName: "team-a", - GitTargetNamespace: "default", - } - - message, options, _, err := pendingWrite.commitMetadata() - require.NoError(t, err) - assert.Equal(t, "reconcile: 2 team-a", message) - assert.Equal(t, DefaultCommitterName, options.Author.Name) - assert.Equal(t, DefaultCommitterName, options.Committer.Name) -} - func TestExecutor_NoOpUnit_SkipsCommit(t *testing.T) { worker, repo, worktree, repoPath := newExecutorTestRepo(t) event := configMapEvent("existing", "alice", "team-a") @@ -266,24 +243,19 @@ func TestMessageSource_MatchesTheMessageActuallyRendered(t *testing.T) { }{ { name: "a live window renders the live template", - write: PendingWrite{Kind: PendingWriteCommit, Events: []Event{makeEvent("alice", "api")}}, + write: PendingWrite{Kind: PendingWriteCommit, Events: []Event{makeEvent("api")}}, want: messageSourceLive, }, { name: "an attached request message is commit_request", write: PendingWrite{ Kind: PendingWriteCommit, - Events: []Event{makeEvent("alice", "api")}, + Events: []Event{makeEvent("api")}, CommitMessage: "fix(api): correct the port", }, want: messageSourceCommitRequest, message: "fix(api): correct the port", }, - { - name: "an atomic snapshot is reconcile", - write: PendingWrite{Kind: PendingWriteAtomic, Events: []Event{makeEvent("alice", "api")}}, - want: messageSourceReconcile, - }, { // A resync arrives pre-rendered, so it must not be mistaken for a literal override. name: "a pre-rendered resync is reconcile, not literal", diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index 572ac6684..2a0ff5d85 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -26,6 +26,11 @@ var ErrResyncSuperseded = errors.New("resync superseded by a newer request for t // the GitTarget once Git is no longer ahead. var ErrResyncDiscarded = errors.New("resync snapshot discarded: an apply made it stale; a fresh one is requested") +// ErrResyncDeferred replies to a resync the worker could not apply yet because the remote could not +// be reached, or the parent branch a new write branch is created from is missing. It is NOT lost: +// it stays pending in its place and is applied with the writes around it once the remote answers. +var ErrResyncDeferred = errors.New("resync deferred until the remote can be reached") + // FinalizeOutcome is the terminal result of resolving a CommitRequest. type FinalizeOutcome string diff --git a/internal/git/commit_request_attach_test.go b/internal/git/commit_request_attach_test.go index 72be19df5..dabbc2e0d 100644 --- a/internal/git/commit_request_attach_test.go +++ b/internal/git/commit_request_attach_test.go @@ -137,7 +137,6 @@ func TestAttach_CommitsOpenWindow(t *testing.T) { configMapTargetEvent("explicit-a", "alice", "team-a"), configMapTargetEvent("explicit-b", "alice", "team-a"), }, - CommitMode: CommitModePerEvent, }}) require.NotNil(t, loop.openWindow, "events should accumulate in an open window") @@ -177,8 +176,7 @@ func TestAttach_EmptyMessageUsesGeneratedMessage(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("solo", "bob", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("solo", "bob", "team-a")}, }}) require.NotNil(t, loop.openWindow) @@ -218,8 +216,7 @@ func TestAttach_CollectGraceJoinsLaterWindow(t *testing.T) { // The work arrives during the grace and opens a window; it must attach. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, }}) loop.endWake(0) require.NotNil(t, loop.openWindow) @@ -255,8 +252,7 @@ func TestAttach_TheAttachStartsTheRequestsMaxDuration(t *testing.T) { attached := time.Now() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) require.NotNil(t, loop.openWindow.pendingCR, "precondition: the request attached to the window") @@ -284,8 +280,7 @@ func TestAttach_AnExpiredWaitNeverTakesALaterWindow(t *testing.T) { forceDue(loop) // the wait ran out, and its timer has not been served yet loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, }}) loop.endWake(0) @@ -314,8 +309,7 @@ func TestAttach_WritesAfterTheClaimJoinOneCommit(t *testing.T) { for _, name := range []string{"first", "second"} { loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent(name, "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent(name, "alice", "team-a")}, }}) loop.endWake(0) _, resolved := outcome(t, worker) @@ -358,8 +352,7 @@ func TestAttach_ForeignWindowIsNotStolen(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) @@ -415,8 +408,7 @@ func TestAttach_ForeignWindowClosingBeforeExpiryIsStillAMismatch(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) @@ -448,8 +440,7 @@ func TestAttach_ClaimedSameAuthorWindowIsNotAMismatch(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) @@ -489,8 +480,7 @@ func TestAttach_ForeignWindowOpeningAfterExpiryIsNotAMismatch(t *testing.T) { // Only now does alice's work arrive and open a window. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) loop.endWake(0) @@ -558,8 +548,7 @@ func TestAttach_AnUnreachableRemoteHoldsTheRequest(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) @@ -587,8 +576,7 @@ func TestFinalizeOpenWindow_ReturnsCommittedFlag(t *testing.T) { assert.False(t, loop.finalizeOpenWindow(), "no open window → false") loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("cm", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) @@ -619,8 +607,7 @@ func TestAttach_NoDiffResolvesAlreadyPresentOnceTheRemoteConfirms(t *testing.T) // First, commit the ConfigMap (no CommitRequest) and push it, so it is already // present in Git. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("present", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("present", "alice", "team-a")}, }}) require.True(t, loop.finalizeOpenWindow()) loop.pushPending() @@ -632,8 +619,7 @@ func TestAttach_NoDiffResolvesAlreadyPresentOnceTheRemoteConfirms(t *testing.T) // A second window re-asserts the SAME object: no diff. Attach a CommitRequest and // finalize it (maxDuration 0s). loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("present", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("present", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) serviceAttach(loop, attachReq("alice", 0)) @@ -682,8 +668,7 @@ func TestAttach_ResyncCutOffCarriesMessageAndResolvesOnPush(t *testing.T) { // Open alice's window with one edit. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) @@ -740,8 +725,7 @@ func TestAttach_ConflictReplayResolvesToPostReplaySHA(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, }}) const message = "save: survive a conflict replay" req := attachReq("alice", 0) @@ -923,8 +907,7 @@ func TestAttach_ReSentAttachDuringPushCooldownDoesNotResolve(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) serviceAttach(loop, attachReq("alice", 0)) @@ -969,8 +952,7 @@ func TestAttach_ReSentAttachDuringAFailedPushDoesNotResolve(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, }}) serviceAttach(loop, attachReq("alice", 0)) require.Len(t, loop.pendingWrites, 1) @@ -1016,8 +998,7 @@ func TestAttach_ACommittedRequestNeverClaimsAnotherWindow(t *testing.T) { const message = "save: only my own window" loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("mine", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("mine", "alice", "team-a")}, }}) req := attachReq("alice", 0) req.Message = message @@ -1027,8 +1008,7 @@ func TestAttach_ACommittedRequestNeverClaimsAnotherWindow(t *testing.T) { // The controller re-sends, and a fresh window of alice's opens while the push is still held. serviceAttach(loop, attachReq("alice", 0)) loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("someone-elses-save", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("someone-elses-save", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) loop.endWake(0) @@ -1052,8 +1032,7 @@ func TestAttach_ShutdownFailsACommittedRequestThatNeverPushed(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("stranded", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("stranded", "alice", "team-a")}, }}) serviceAttach(loop, attachReq("alice", 0)) require.Len(t, loop.pendingWrites, 1) diff --git a/internal/git/commit_request_withdraw_test.go b/internal/git/commit_request_withdraw_test.go index 44175c3a4..f2ffa423c 100644 --- a/internal/git/commit_request_withdraw_test.go +++ b/internal/git/commit_request_withdraw_test.go @@ -40,8 +40,7 @@ func TestWithdraw_AWaitingRequestIsCancelledForGood(t *testing.T) { // A late re-send of the attach, and a window it would have attached to, change nothing. serviceAttach(loop, req) loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow) assert.Nil(t, loop.openWindow.pendingCR, "a withdrawn request never attaches") @@ -76,8 +75,7 @@ func TestWithdraw_AnAttachedRequestIsHeldAndStillCommits(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, }}) req := attachReq("alice", time.Hour) serviceAttach(loop, req) diff --git a/internal/git/commit_test.go b/internal/git/commit_test.go index 0ac0adbac..3dc864f21 100644 --- a/internal/git/commit_test.go +++ b/internal/git/commit_test.go @@ -108,14 +108,10 @@ func TestRenderEventCommitMessage_CustomTemplate(t *testing.T) { assert.Equal(t, "audit(platform): alice UPDATE prod/api", message) } -func TestRenderReconcileCommitMessageFromEvents_DefaultTemplate(t *testing.T) { - // The events-based path carries no type or revision, so the default falls back to the +func TestRenderReconcileCommitMessage_DefaultTemplateForTheWholeTarget(t *testing.T) { + // A whole-GitTarget reconcile carries no type or revision, so the default falls back to the // type-less subject with no resourceVersion suffix. - message, err := renderReconcileCommitMessageFromEvents( - []Event{{Operation: "CREATE"}, {Operation: "DELETE"}}, - "demo", - ResolveCommitConfig(nil), - ) + message, err := renderReconcileCommitMessage(2, "demo", nil, "", ResolveCommitConfig(nil)) require.NoError(t, err) assert.Equal(t, "chore: reconcile 2 resources", message) } @@ -261,7 +257,7 @@ func TestRenderGroupCommitMessage_CustomTemplate(t *testing.T) { func TestCommitOptionsFor_EmptyAuthorFallsThroughToCommitter(t *testing.T) { config := ResolveCommitConfig(nil) pendingWrite := PendingWrite{ - Kind: PendingWriteAtomic, + Kind: PendingWriteCommit, } when := time.Now() diff --git a/internal/git/commit_window_metrics_test.go b/internal/git/commit_window_metrics_test.go index 640f6ef10..e58ce24ca 100644 --- a/internal/git/commit_window_metrics_test.go +++ b/internal/git/commit_window_metrics_test.go @@ -82,8 +82,7 @@ func TestCommitWindowMetrics_EachCloseReasonIsCountedUnderItsOwnName(t *testing. // identity_change: another author's write arrives while alice's window is open. writeTo(loop, "b") loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("c", "bob", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("c", "bob", "team-a")}, }}) // shutdown: bob's window is open when the worker stops. diff --git a/internal/git/commit_window_timers_test.go b/internal/git/commit_window_timers_test.go index b5922b619..38e27f5ed 100644 --- a/internal/git/commit_window_timers_test.go +++ b/internal/git/commit_window_timers_test.go @@ -21,8 +21,7 @@ import ( // names another. func writeTo(loop *branchWorkerEventLoop, name string) { loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent(name, "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent(name, "alice", "team-a")}, }}) } diff --git a/internal/git/content_writer.go b/internal/git/content_writer.go index a8b2b061d..b382db666 100644 --- a/internal/git/content_writer.go +++ b/internal/git/content_writer.go @@ -44,7 +44,7 @@ type contentWriter struct { } type eventContentWriter interface { - buildContentForWrite(ctx context.Context, event Event) ([]byte, error) + buildContentForWrite(ctx context.Context, target placementTarget, event Event) ([]byte, error) filePathForIdentifier(id types.ResourceIdentifier) string isSensitiveIdentifier(id types.ResourceIdentifier) bool } @@ -65,8 +65,11 @@ func (w *contentWriter) setEncryptor(encryptor Encryptor, scope string) { } // buildContentForWrite renders event content to stable ordered YAML and applies -// sensitive-resource encryption when configured. -func (w *contentWriter) buildContentForWrite(ctx context.Context, event Event) ([]byte, error) { +// sensitive-resource encryption when configured. target is the GitTarget the write is for, which +// secret_encryptions_total is labelled by. +func (w *contentWriter) buildContentForWrite( + ctx context.Context, target placementTarget, event Event, +) ([]byte, error) { content, err := sanitize.MarshalToOrderedYAML(event.Object) if err != nil { return nil, fmt.Errorf("failed to marshal object to YAML: %w", err) @@ -76,7 +79,7 @@ func (w *contentWriter) buildContentForWrite(ctx context.Context, event Event) ( return content, nil } - return w.encryptSensitiveContent(ctx, event, content) + return w.encryptSensitiveContent(ctx, target, event, content) } func (w *contentWriter) filePathForIdentifier(id types.ResourceIdentifier) string { @@ -87,7 +90,9 @@ func (w *contentWriter) isSensitiveIdentifier(id types.ResourceIdentifier) bool return w.sensitiveResources.IsSensitive(id.Group, id.Resource) } -func (w *contentWriter) encryptSensitiveContent(ctx context.Context, event Event, plain []byte) ([]byte, error) { +func (w *contentWriter) encryptSensitiveContent( + ctx context.Context, target placementTarget, event Event, plain []byte, +) ([]byte, error) { meta := buildResourceMeta(event) identityKey := meta.Identifier.Key() digest := sha256.Sum256(plain) @@ -110,7 +115,7 @@ func (w *contentWriter) encryptSensitiveContent(ctx context.Context, event Event cacheKey := fmt.Sprintf("%s:%x", scopedIdentityKey, digest) w.mu.RLock() - cached, ok := w.cachedEncryptedContent(ctx, scopedIdentityKey, cacheKey, currentMarker) + cached, ok := w.cachedEncryptedContent(ctx, target, scopedIdentityKey, cacheKey, currentMarker) w.mu.RUnlock() if ok { return cached, nil @@ -122,10 +127,10 @@ func (w *contentWriter) encryptSensitiveContent(ctx context.Context, event Event encrypted, err := encryptor.Encrypt(ctx, plain, ResourceMeta(meta)) if err != nil { - recordSecretEncryption(ctx, secretEncryptionFailed) + recordSecretEncryption(ctx, target, secretEncryptionFailed) return nil, fmt.Errorf("secret encryption failed: %w", err) } - recordSecretEncryption(ctx, secretEncryptionEncrypted) + recordSecretEncryption(ctx, target, secretEncryptionEncrypted) w.mu.Lock() w.encryptedCache[cacheKey] = append([]byte(nil), encrypted...) @@ -137,6 +142,7 @@ func (w *contentWriter) encryptSensitiveContent(ctx context.Context, event Event func (w *contentWriter) cachedEncryptedContent( ctx context.Context, + target placementTarget, identityKey, cacheKey string, currentMarker sensitiveMarker, ) ([]byte, bool) { @@ -148,7 +154,7 @@ func (w *contentWriter) cachedEncryptedContent( if !ok { return nil, false } - recordSecretEncryption(ctx, secretEncryptionCached) + recordSecretEncryption(ctx, target, secretEncryptionCached) return append([]byte(nil), cached...), true } @@ -181,9 +187,13 @@ const ( // `cache_hits` and `marker_skips` were incremented on consecutive lines of one branch and could // never differ. A share of one total is also the ratio the old "cache effectiveness" query was // reaching for, which divided two counters over disjoint populations. -func recordSecretEncryption(ctx context.Context, outcome string) { +// +// It names the GitTarget, so "whose Secrets are being rejected" is a selector rather than a log +// search. A batch built by a test carries no target and publishes the empty pair. +func recordSecretEncryption(ctx context.Context, target placementTarget, outcome string) { if telemetry.SecretEncryptionsTotal == nil { return } - telemetry.SecretEncryptionsTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", outcome))) + telemetry.SecretEncryptionsTotal.Add(ctx, 1, + metric.WithAttributes(append(target.attrs(), attribute.String("outcome", outcome))...)) } diff --git a/internal/git/content_writer_test.go b/internal/git/content_writer_test.go index c2aed7ee5..65d71f03b 100644 --- a/internal/git/content_writer_test.go +++ b/internal/git/content_writer_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" "github.com/ConfigButler/gitops-reverser/internal/types" ) @@ -47,7 +48,7 @@ func TestBuildContentForWrite_MarshalOrderedYAML(t *testing.T) { }, } - got, err := writer.buildContentForWrite(context.Background(), event) + got, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) output := string(got) @@ -79,7 +80,7 @@ func TestBuildContentForWrite_ReturnsMarshalError(t *testing.T) { }, } - _, err := writer.buildContentForWrite(context.Background(), event) + _, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.Error(t, err) assert.Contains(t, err.Error(), "failed to marshal object to YAML") } @@ -110,7 +111,7 @@ func TestBuildContentForWrite_SecretRequiresEncryptor(t *testing.T) { }, } - _, err := writer.buildContentForWrite(context.Background(), event) + _, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.Error(t, err) assert.Contains(t, err.Error(), "secret encryption is required but no encryptor is configured") } @@ -121,7 +122,7 @@ func TestBuildContentForWrite_AdditionalSensitiveResourceRequiresEncryptor(t *te writer := newContentWriter(policy) event := tenantSecretEvent("v1beta1") - _, err = writer.buildContentForWrite(context.Background(), event) + _, err = writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.Error(t, err) assert.Contains(t, err.Error(), "secret encryption is required but no encryptor is configured") } @@ -133,7 +134,7 @@ func TestBuildContentForWrite_AdditionalSensitiveResourceEncrypts(t *testing.T) enc := &stubEncryptor{result: []byte("encrypted tenant secret\n")} writer.setEncryptor(enc, "test-scope") - got, err := writer.buildContentForWrite(context.Background(), tenantSecretEvent("v1alpha1")) + got, err := writer.buildContentForWrite(context.Background(), placementTarget{}, tenantSecretEvent("v1alpha1")) require.NoError(t, err) assert.Equal(t, []byte("encrypted tenant secret\n"), got) assert.Equal(t, 1, enc.callCount) @@ -173,9 +174,9 @@ func TestBuildContentForWrite_SecretEncryptionCacheMarkerReuse(t *testing.T) { Object: obj, } - first, err := writer.buildContentForWrite(context.Background(), event) + first, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) - second, err := writer.buildContentForWrite(context.Background(), event) + second, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) assert.Equal(t, 1, enc.callCount) assert.Equal(t, first, second) @@ -216,11 +217,11 @@ func TestBuildContentForWrite_SecretUIDChangeForcesReencrypt(t *testing.T) { }, Object: makeSecret("uid-1"), } - _, err := writer.buildContentForWrite(context.Background(), event) + _, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) event.Object = makeSecret("uid-2") - _, err = writer.buildContentForWrite(context.Background(), event) + _, err = writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) assert.Equal(t, 2, enc.callCount) } @@ -253,7 +254,7 @@ func TestBuildContentForWrite_SecretEncryptionFailure(t *testing.T) { }, } - _, err := writer.buildContentForWrite(context.Background(), event) + _, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.Error(t, err) assert.Contains(t, err.Error(), "secret encryption failed") } @@ -287,14 +288,14 @@ func TestBuildContentForWrite_SecretEncryptionScopeChangeForcesReencrypt(t *test }, } - _, err := writer.buildContentForWrite(context.Background(), event) + _, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) - _, err = writer.buildContentForWrite(context.Background(), event) + _, err = writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) assert.Equal(t, 1, enc.callCount, "same scope should reuse cached encrypted content") writer.setEncryptor(enc, "scope-b") - _, err = writer.buildContentForWrite(context.Background(), event) + _, err = writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) assert.Equal(t, 2, enc.callCount, "scope change should force re-encryption") } @@ -323,3 +324,46 @@ func tenantSecretEvent(version string) Event { }, } } + +// secret_encryptions_total names the GitTarget the write was for, so a rejected Secret write is +// found by selector: a failed encryption, and then the cache hit on a later identical write. +func TestBuildContentForWrite_CountsEncryptionsPerGitTarget(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + writer := newContentWriter(types.SensitiveResourcePolicy{}) + target := placementTarget{namespace: "team-a", name: "vault"} + labels := func(outcome string) map[string]string { + return map[string]string{"gittarget_namespace": "team-a", "gittarget_name": "vault", "outcome": outcome} + } + + writer.setEncryptor(&stubEncryptor{err: errors.New("no key")}, "scope") + _, err = writer.buildContentForWrite(context.Background(), target, secretEventWithUID("uid-1")) + require.Error(t, err) + failed, ok := telemetry.CollectInt64Sum(reader, "gitopsreverser_secret_encryptions_total", labels("failed")) + require.True(t, ok) + assert.Equal(t, int64(1), failed) + + writer.setEncryptor(&stubEncryptor{result: []byte("encrypted: true\n")}, "scope") + for range 2 { + _, err = writer.buildContentForWrite(context.Background(), target, secretEventWithUID("uid-1")) + require.NoError(t, err) + } + cached, ok := telemetry.CollectInt64Sum(reader, "gitopsreverser_secret_encryptions_total", labels("cached")) + require.True(t, ok) + assert.Equal(t, int64(1), cached) +} + +// secretEventWithUID is a v1 Secret write carrying the identity the encryption cache keys on. +func secretEventWithUID(uid string) Event { + return Event{ + Identifier: types.ResourceIdentifier{Version: "v1", Resource: "secrets", Namespace: "default", Name: "s"}, + Object: &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]interface{}{ + "name": "s", "namespace": "default", "uid": uid, "resourceVersion": "10", + }, + "data": map[string]interface{}{"password": "cGxhaW4="}, + }}, + } +} diff --git a/internal/git/credentials.go b/internal/git/credentials.go index 3f3e60a59..d02aa431e 100644 --- a/internal/git/credentials.go +++ b/internal/git/credentials.go @@ -164,23 +164,6 @@ func (c Credential) Options() []gitclient.Option { } } -// AuthFromSecretData resolves go-git transport options from an already-fetched Git credentials -// Secret. It is the thin wrapper over CredentialFromSecretData; callers that need to know what kind -// of credential was produced should use that instead. -func AuthFromSecretData( - ctx context.Context, - k8sClient client.Client, - provider *v1alpha3.GitProvider, - secret *corev1.Secret, - hostKeys SSHHostKeyConfig, -) ([]gitclient.Option, error) { - cred, err := CredentialFromSecretData(ctx, k8sClient, provider, secret, hostKeys) - if err != nil { - return nil, err - } - return CredentialOptionsForProvider(provider, cred, CredentialTransportPolicy{}) -} - // CredentialFromSecretData resolves a credential from an already-fetched Git credentials Secret, // accepting the Kubernetes-native, Flux, and Argo CD key dialects (the credentials Secret is the // one portable artifact across those ecosystems). provider supplies the namespace and the optional diff --git a/internal/git/credentials_test.go b/internal/git/credentials_test.go index fadb7b06e..279869c19 100644 --- a/internal/git/credentials_test.go +++ b/internal/git/credentials_test.go @@ -47,9 +47,9 @@ func credTestClient(t *testing.T, objs ...client.Object) client.Client { return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() } -// AuthFromSecretData must read the SSH private key under any of the three ecosystem dialects +// CredentialFromSecretData must read the SSH private key under any of the three ecosystem dialects // (Kubernetes-native ssh-privatekey, Flux identity, Argo CD sshPrivateKey). -func TestAuthFromSecretData_SSHKeyDialects(t *testing.T) { +func TestCredentialFromSecretData_SSHKeyDialects(t *testing.T) { privateKey, knownHosts := credTestSSHKey(t) c := credTestClient(t) for _, keyName := range []string{"ssh-privatekey", "identity", "sshPrivateKey"} { @@ -68,7 +68,7 @@ func TestAuthFromSecretData_SSHKeyDialects(t *testing.T) { // A password presented alongside an SSH key is its passphrase (Flux's shared-key convention), not // HTTP basic auth — so the result is still SSH public-key auth, never BasicAuth. -func TestAuthFromSecretData_PasswordIsSSHPassphraseWhenKeyPresent(t *testing.T) { +func TestCredentialFromSecretData_PasswordIsSSHPassphraseWhenKeyPresent(t *testing.T) { privateKey, knownHosts := credTestSSHKey(t) c := credTestClient(t) secret := &corev1.Secret{Data: map[string][]byte{ @@ -82,7 +82,7 @@ func TestAuthFromSecretData_PasswordIsSSHPassphraseWhenKeyPresent(t *testing.T) assert.NotNil(t, auth.SSH) } -func TestAuthFromSecretData_HTTPBasicAndBearer(t *testing.T) { +func TestCredentialFromSecretData_HTTPBasicAndBearer(t *testing.T) { c := credTestClient(t) t.Run("basic", func(t *testing.T) { diff --git a/internal/git/decided_write_test.go b/internal/git/decided_write_test.go index a68c1f906..806c74ac8 100644 --- a/internal/git/decided_write_test.go +++ b/internal/git/decided_write_test.go @@ -59,8 +59,7 @@ func TestDecidedWrite_SurvivesAFailedRebuildAndLandsThroughTheRetry(t *testing.T // One write is committed and retained, and then the checkout has to be rebuilt before anything // can commit on it, while the remote is unreachable. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("retained", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("retained", "alice", ledgerTargetName)}, }}) require.True(t, loop.finalizeOpenWindow()) f.worker.markWorktreeDirty("a write failed part-way and could not be undone") @@ -68,8 +67,7 @@ func TestDecidedWrite_SurvivesAFailedRebuildAndLandsThroughTheRetry(t *testing.T // A save's window closes during the outage. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("saved-during-outage", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("saved-during-outage", "alice", ledgerTargetName)}, }}) req := attachReq("alice", 0) req.GitTargetName = ledgerTargetName @@ -86,8 +84,7 @@ func TestDecidedWrite_SurvivesAFailedRebuildAndLandsThroughTheRetry(t *testing.T // Another window during the outage is decided without spending a connection on the rebuild. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("also-during-outage", "bob", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("also-during-outage", "bob", ledgerTargetName)}, }}) require.True(t, loop.finalizeOpenWindow()) assert.Equal(t, int32(1), syncs.Load(), "a decision waits for the pending retry instead of fetching") @@ -122,7 +119,7 @@ func TestDecidedWrite_ADeferredDeleteObeysATightenedPrunePolicy(t *testing.T) { loop := newBranchWorkerEventLoop(f.worker, time.Hour) defer loop.stopTimers() write := func(event Event) { - loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}}}) require.True(t, loop.finalizeOpenWindow()) } @@ -170,8 +167,7 @@ func TestDecidedWrite_AdmissionClosesAtTheBudgetDuringAnOutage(t *testing.T) { defer loop.stopTimers() write := func(name string) { loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent(name, "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent(name, "alice", ledgerTargetName)}, }}) loop.finalizeOpenWindow() // the budget usually closed it on arrival already loop.publishLoopState(0) // what every loop iteration does @@ -270,8 +266,7 @@ func TestDecidedWrite_AResyncDuringBackoffWaitsForTheRetry(t *testing.T) { require.ErrorContains(t, resync(), "connection refused", "the caller hears the known failure at once") loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("behind-the-resyncs", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("behind-the-resyncs", "alice", ledgerTargetName)}, }}) loop.finalizeOpenWindow() assert.Equal(t, int32(1), calls.Load(), "nothing dials the remote before the retry is due") @@ -299,7 +294,7 @@ func TestDecidedWrite_SameObjectEditsReplayAsDecidedOntoARemoteHoldingTheLast(t edit := func(value string) []byte { event := configMapTargetEvent("settings", "alice", ledgerTargetName) event.Object.Object["data"] = map[string]any{"value": value} - loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}}}) path := filepath.Join(f.worker.repoPath(), "team-a", generateFilePath(event.Identifier, itypes.SensitiveResourcePolicy{})) content, err := os.ReadFile(path) diff --git a/internal/git/dirty_worktree_recovery_test.go b/internal/git/dirty_worktree_recovery_test.go index 27503fd08..4bd07f755 100644 --- a/internal/git/dirty_worktree_recovery_test.go +++ b/internal/git/dirty_worktree_recovery_test.go @@ -81,56 +81,6 @@ func headTreeContains(t *testing.T, repoPath, path string) bool { return true } -// TestAtomicWrite_DoesNotCommitAFailedWritesLeftovers is the §3.1 interleaving driven through the -// ATOMIC path, which had no recovery at all. -// -// handleAtomicRequest finalizes the open window first, and that finalize is where recovery used to -// be reached — but finalizeOpenWindowWithReason returns at its first line when no window is open, -// which is the ordinary case for an atomic write. So the atomic planned on a worktree still -// holding the failed write's staged file and committed it under its own author. -func TestAtomicWrite_DoesNotCommitAFailedWritesLeftovers(t *testing.T) { - worker, _, _ := setupCommitPushSplitWorker(t) - createPlainGitTarget(t, worker, "team-a", "team-a") - - _, err := worker.getGitProvider(worker.ctx) - require.NoError(t, err) - repoPath := worker.repoPath() - - // The cooldown is held, so write A commits locally and stays retained. That retention is what - // makes commitPendingWrites' own reset unavailable. - loop := newBranchWorkerEventLoop(worker, time.Hour) - loop.lastPushAt = time.Now() - defer loop.stopTimers() - - // 1. Write A commits and is retained. - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("write-a", "alice", "team-a")}, - CommitMode: CommitModePerEvent, - }}) - require.True(t, loop.finalizeOpenWindow()) - require.Len(t, loop.pendingWrites, 1) - - // 2. Write B fails part-way through its batch, leaving its file staged. - stageLeftoverFromAFailedWrite(t, worker) - - // 3. An atomic write C arrives with no window open. - require.Nil(t, loop.openWindow) - loop.handleAtomicRequest(&WriteRequest{ - Events: []Event{configMapEvent("write-c", "carol", "")}, - CommitMode: CommitModeAtomic, - GitTargetName: "team-a", - GitTargetNamespace: "default", - }) - loop.endWake(0) - - require.Len(t, loop.pendingWrites, 2, "C committed: A is retained alongside it") - assert.False(t, headTreeContains(t, repoPath, leftoverPath), - "C's commit must not carry the file a failed write left staged") - assert.True(t, headTreeContains(t, repoPath, "team-a/default/configmaps/write-a.yaml"), - "the replay must rebuild A: a recovery resets, and re-plans rather than discarding work") - assert.False(t, worker.worktreeDirty(), "the reset inside the recovery clears the flag") -} - // TestEveryLoopCommitPathRecoversADirtyWorktree enumerates the loop's commit entry points against // one dirty fixture. // @@ -147,25 +97,11 @@ func TestEveryLoopCommitPathRecoversADirtyWorktree(t *testing.T) { commit: func(t *testing.T, loop *branchWorkerEventLoop) { t.Helper() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("from-window", "bob", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("from-window", "bob", "team-a")}, }}) require.True(t, loop.finalizeOpenWindow()) }, }, - { - name: "atomic request", - commit: func(t *testing.T, loop *branchWorkerEventLoop) { - t.Helper() - loop.handleAtomicRequest(&WriteRequest{ - Events: []Event{configMapEvent("from-atomic", "carol", "")}, - CommitMode: CommitModeAtomic, - GitTargetName: "team-a", - GitTargetNamespace: "default", - }) - loop.endWake(0) - }, - }, { name: "resync", commit: func(t *testing.T, loop *branchWorkerEventLoop) { @@ -210,8 +146,7 @@ func TestEveryLoopCommitPathRecoversADirtyWorktree(t *testing.T) { // Retained work, so commitPendingWrites cannot reset for us. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("retained", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("retained", "alice", "team-a")}, }}) require.True(t, loop.finalizeOpenWindow()) require.Len(t, loop.pendingWrites, 1) diff --git a/internal/git/git_path_refusal.go b/internal/git/git_path_refusal.go index 58021895f..920ec8be2 100644 --- a/internal/git/git_path_refusal.go +++ b/internal/git/git_path_refusal.go @@ -112,21 +112,3 @@ func (w *BranchWorker) reportPathRefusal( } return true, refused } - -// atomicRefusalTarget names the GitTarget an atomic request writes for. Request-level target -// metadata is authoritative when set — buildAtomicPendingWrite resolves it and stamps it onto -// every event — but it fills that metadata only when the request carries it, leaving requests -// whose events name their own target. So fall back to the events rather than reporting a -// refusal against an empty reference. Returns the target's (name, namespace), both empty when -// nothing names a target — which reportPathRefusal treats as unattributable. -func atomicRefusalTarget(request *WriteRequest) (string, string) { - if request.GitTargetName != "" && request.GitTargetNamespace != "" { - return request.GitTargetName, request.GitTargetNamespace - } - for _, event := range request.Events { - if event.GitTargetName != "" && event.GitTargetNamespace != "" { - return event.GitTargetName, event.GitTargetNamespace - } - } - return "", "" -} diff --git a/internal/git/git_path_refusal_test.go b/internal/git/git_path_refusal_test.go index 25a290950..53ebc5e74 100644 --- a/internal/git/git_path_refusal_test.go +++ b/internal/git/git_path_refusal_test.go @@ -100,55 +100,3 @@ func TestReportPathRefusal_UnattributableRefusalIsNotRecorded(t *testing.T) { "a refusal with an incomplete target reference must never be recorded (%q/%q)", c.ns, c.name) } } - -// TestAtomicRefusalTarget_PrefersRequestThenEvents proves the atomic path names the right -// GitTarget. buildAtomicPendingWrite stamps request-level metadata onto every event, but only -// when the request carries it — a request whose events name their own target would otherwise -// have its refusal reported against an empty reference. -func TestAtomicRefusalTarget_PrefersRequestThenEvents(t *testing.T) { - eventFor := func(name, namespace string) Event { - return Event{GitTargetName: name, GitTargetNamespace: namespace} - } - - cases := []struct { - name string - request *WriteRequest - wantName string - wantNamespace string - }{ - { - name: "request-level metadata wins", - request: &WriteRequest{ - GitTargetName: "from-request", GitTargetNamespace: "team-a", - Events: []Event{eventFor("from-event", "team-b")}, - }, - wantName: "from-request", wantNamespace: "team-a", - }, - { - name: "falls back to the first event that names a target", - request: &WriteRequest{ - Events: []Event{eventFor("", ""), eventFor("from-event", "team-b")}, - }, - wantName: "from-event", wantNamespace: "team-b", - }, - { - name: "a half-set request falls back rather than reporting a partial reference", - request: &WriteRequest{ - GitTargetName: "from-request", - Events: []Event{eventFor("from-event", "team-b")}, - }, - wantName: "from-event", wantNamespace: "team-b", - }, - { - name: "nothing names a target", - request: &WriteRequest{Events: []Event{eventFor("", "")}}, - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - name, namespace := atomicRefusalTarget(c.request) - assert.Equal(t, c.wantName, name) - assert.Equal(t, c.wantNamespace, namespace) - }) - } -} diff --git a/internal/git/git_roundtrip_ledger_test.go b/internal/git/git_roundtrip_ledger_test.go index 7dc2023b2..83e1924fe 100644 --- a/internal/git/git_roundtrip_ledger_test.go +++ b/internal/git/git_roundtrip_ledger_test.go @@ -205,21 +205,21 @@ type ledgerOp struct { func ledgerOperations() []ledgerOp { return []ledgerOp{ { - name: "1. bootstrap helper, empty remote", + name: "1. first clone, empty remote", slug: "start-empty", seeded: false, run: func(f *ledgerFixture) { f.createLedgerTarget("team-a", nil) - require.NoError(f.t, f.worker.EnsurePathBootstrapped("team-a", "target-a", "default")) + require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) }, }, { - name: "2. bootstrap helper, populated remote", + name: "2. first clone, populated remote", slug: "start-populated", seeded: true, run: func(f *ledgerFixture) { f.createLedgerTarget("team-a", nil) - require.NoError(f.t, f.worker.EnsurePathBootstrapped("team-a", "target-a", "default")) + require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) }, }, { diff --git a/internal/git/helpers.go b/internal/git/helpers.go index abbdae2eb..fe7186a8d 100644 --- a/internal/git/helpers.go +++ b/internal/git/helpers.go @@ -16,15 +16,6 @@ import ( "github.com/ConfigButler/gitops-reverser/api/v1alpha3" ) -// GetCommitSigner fetches commit signing material from the specified secret. -func GetCommitSigner( - ctx context.Context, - k8sClient client.Client, - provider *v1alpha3.GitProvider, -) (gogit.Signer, error) { - return getCommitSigner(ctx, k8sClient, provider) -} - func getCommitSigner( ctx context.Context, k8sClient client.Client, diff --git a/internal/git/helpers_test.go b/internal/git/helpers_test.go index 796a1227e..aafc077eb 100644 --- a/internal/git/helpers_test.go +++ b/internal/git/helpers_test.go @@ -241,8 +241,8 @@ func newTestBranchWorker( return worker, nil } -// makeEvent builds a ConfigMap event for commit planning tests. -func makeEvent(author, name string) Event { +// makeEvent builds a ConfigMap event by alice for commit planning tests. +func makeEvent(name string) Event { const target = "team-a" return Event{ @@ -265,7 +265,7 @@ func makeEvent(author, name string) Event { "data": map[string]interface{}{"v": "v1"}, }, }, - UserInfo: UserInfo{Username: author}, + UserInfo: UserInfo{Username: "alice"}, Path: "team-" + target, GitTargetName: target, GitTargetNamespace: "default", diff --git a/internal/git/inplace_edit_test.go b/internal/git/inplace_edit_test.go index e95df07db..1ddf52a34 100644 --- a/internal/git/inplace_edit_test.go +++ b/internal/git/inplace_edit_test.go @@ -173,7 +173,7 @@ func TestPlanFlush_AppliesChangeToCanonicalFile(t *testing.T) { // Seed with the canonical rendering of a *different* value, so the update is a // real change against an operator-canonical file. - seedCanonical, err := writer.buildContentForWrite(context.Background(), inplaceCMEvent("blue")) + seedCanonical, err := writer.buildContentForWrite(context.Background(), placementTarget{}, inplaceCMEvent("blue")) require.NoError(t, err) require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o750)) require.NoError(t, os.WriteFile(full, seedCanonical, 0o600)) @@ -199,7 +199,7 @@ func TestPlanFlush_IdenticalUpdateIsNoOp(t *testing.T) { relPath := writer.filePathForIdentifier(event.Identifier) full := filepath.Join(root, relPath) - seed, err := writer.buildContentForWrite(context.Background(), event) + seed, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o750)) require.NoError(t, os.WriteFile(full, seed, 0o600)) diff --git a/internal/git/intake.go b/internal/git/intake.go index 7d3147e09..f021ff618 100644 --- a/internal/git/intake.go +++ b/internal/git/intake.go @@ -123,7 +123,7 @@ func (w *BranchWorker) intakeRefusal(charge int64) error { return errAdmissionClosed } -// eventCharge is what a queued live or atomic write is charged: what keeping each of its events +// eventCharge is what a queued live write is charged: what keeping each of its events // costs, as a window or a decided write keeps it. func eventCharge(request *WriteRequest) int64 { var charge int64 diff --git a/internal/git/intake_test.go b/internal/git/intake_test.go index 7ab1e63d9..7a7854fa0 100644 --- a/internal/git/intake_test.go +++ b/internal/git/intake_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" itypes "github.com/ConfigButler/gitops-reverser/internal/types" @@ -32,8 +33,7 @@ func outageLoop(t *testing.T, slug string) (*ledgerFixture, *branchWorkerEventLo loop := newBranchWorkerEventLoop(f.worker, time.Hour) t.Cleanup(loop.stopTimers) loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held-during-the-outage", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("held-during-the-outage", "alice", ledgerTargetName)}, }}) loop.finalizeOpenWindow() loop.publishLoopState(0) @@ -229,7 +229,7 @@ func TestIntake_AWindowChargesAReplacedEventOnce(t *testing.T) { _, loop, _ := outageLoop(t, "intake-window-replaced") edit := func() { loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{deleteEvent("edited-twice")}, CommitMode: CommitModePerEvent, + Events: []Event{deleteEvent("edited-twice")}, }}) } edit() @@ -303,3 +303,23 @@ func TestEnqueueResync_ACoalescedSnapshotIsNotKeptByTheQueue(t *testing.T) { assert.Equal(t, queued, reachable, "the queue keeps exactly the snapshots it is charged for") assert.LessOrEqual(t, loop.heldBytes()+reachable, f.worker.branchBufferMaxBytes) } + +// A resync the worker cannot apply while the remote is unreachable is answered as deferred, not +// failed: its caller hears ErrResyncDeferred, and the resync itself stays pending in its place. +func TestIntake_AResyncDuringAnOutageIsDeferred(t *testing.T) { + _, loop, _ := outageLoop(t, "resync-deferred") + scope := ResyncScopeFor(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, "") + resultCh := make(chan ResyncResult, 1) + loop.handleResyncRequest(&ResyncRequest{ + GitTargetName: ledgerTargetName, GitTargetNamespace: "default", Scope: &scope, Result: resultCh, + }) + fireRetry(loop) + + select { + case result := <-resultCh: + require.ErrorIs(t, result.Err, ErrResyncDeferred) + default: + t.Fatal("the resync's caller was not answered") + } + assert.Len(t, loop.pendingWrites, 2, "the deferred resync stays pending behind the held write") +} diff --git a/internal/git/known_placement_bugs_test.go b/internal/git/known_placement_bugs_test.go index 0b49ac922..5723581dc 100644 --- a/internal/git/known_placement_bugs_test.go +++ b/internal/git/known_placement_bugs_test.go @@ -104,7 +104,7 @@ func TestPlanFlush_NoOpInMultiDocReportsNoChange(t *testing.T) { full := filepath.Join(root, placedManifestPath) // The target document, byte-for-byte as the operator would render it. - targetDoc, err := writer.buildContentForWrite(context.Background(), event) + targetDoc, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) // An unrelated, editable resource as the first document of the file. diff --git a/internal/git/literal_message_test.go b/internal/git/literal_message_test.go index f2c2cd987..c8a5a17ad 100644 --- a/internal/git/literal_message_test.go +++ b/internal/git/literal_message_test.go @@ -37,17 +37,14 @@ func TestValidateLiteralCommitMessage_Contract(t *testing.T) { } func TestCommitMetadata_LiteralPrecedenceAndPreservation(t *testing.T) { - for _, kind := range []PendingWriteKind{PendingWriteCommit, PendingWriteAtomic} { - for _, message := range []string{" fix: {{.Author}}\n\nbody ", strings.Repeat("ü", 1024)} { - p := PendingWrite{Kind: kind, Events: []Event{makeEvent("alice", "api")}, CommitMessage: message, - CommitConfig: ResolveCommitConfig(nil)} - p.CommitConfig.Message.LiveTemplate = "{{ invalid" - p.CommitConfig.Message.ReconcileTemplate = "{{ invalid" - actual, options, _, err := p.commitMetadata() - require.NoError(t, err) - assert.Equal(t, message, actual) - assert.Equal(t, DefaultCommitterName, options.Committer.Name) - } + for _, message := range []string{" fix: {{.Author}}\n\nbody ", strings.Repeat("ü", 1024)} { + p := PendingWrite{Kind: PendingWriteCommit, Events: []Event{makeEvent("api")}, CommitMessage: message, + CommitConfig: ResolveCommitConfig(nil)} + p.CommitConfig.Message.LiveTemplate = "{{ invalid" + actual, options, _, err := p.commitMetadata() + require.NoError(t, err) + assert.Equal(t, message, actual) + assert.Equal(t, DefaultCommitterName, options.Committer.Name) } p := PendingWrite{Kind: PendingWriteCommit, CommitMessage: " \n ", CommitConfig: ResolveCommitConfig(nil)} _, _, _, err := p.commitMetadata() diff --git a/internal/git/parent_recovery_test.go b/internal/git/parent_recovery_test.go index 57a87d2f0..0abc2e83c 100644 --- a/internal/git/parent_recovery_test.go +++ b/internal/git/parent_recovery_test.go @@ -147,7 +147,7 @@ func TestParentRecovery_WritesDecidedWhileTheParentIsMissingAreKept(t *testing.T write := func(target, name string) { event := configMapTargetEvent(name, "alice", target) - loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}}}) } write(newBranchTarget, "a1") write("other-target", "b1") @@ -234,7 +234,7 @@ func TestParentRecovery_ProbeBudgetIsPerWorker(t *testing.T) { }) event := configMapTargetEvent("cm-"+target, "alice", target) f.loop.handleQueueItem( - WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}, + WorkItem{Request: &WriteRequest{Events: []Event{event}}}, ) } } diff --git a/internal/git/pending_writes.go b/internal/git/pending_writes.go index ddc9234b6..f3abce4e6 100644 --- a/internal/git/pending_writes.go +++ b/internal/git/pending_writes.go @@ -47,68 +47,6 @@ func (w *BranchWorker) buildGroupedPendingWrite(ctx context.Context, events []Ev }, nil } -func (w *BranchWorker) buildAtomicPendingWrite(ctx context.Context, request *WriteRequest) (*PendingWrite, error) { - if request == nil { - return nil, errors.New("write request is required") - } - if len(request.Events) == 0 { - return nil, errors.New("atomic pending write requires at least one event") - } - - provider, err := w.getGitProvider(ctx) - if err != nil { - return nil, fmt.Errorf("get GitProvider: %w", err) - } - - signer, err := getCommitSigner(ctx, w.Client, provider) - if err != nil { - return nil, fmt.Errorf("resolve signer: %w", err) - } - - commitConfig := ResolveCommitConfig(provider.Spec.Commit) - resolvedEvents := append([]Event(nil), request.Events...) - targets := map[pendingTargetKey]ResolvedTargetMetadata{} - - if request.GitTargetName != "" && request.GitTargetNamespace != "" { - targetMetadata, err := w.resolveTargetMetadata( - ctx, - request.GitTargetName, - request.GitTargetNamespace, - ) - if err != nil { - return nil, err - } - - targetKey := pendingTargetKey{ - Name: targetMetadata.Name, - Namespace: targetMetadata.Namespace, - } - targets[targetKey] = targetMetadata - commitConfig = commitConfig.WithTargetMessage(targetMetadata.CommitMessage) - - for i := range resolvedEvents { - if resolvedEvents[i].Path == "" { - resolvedEvents[i].Path = targetMetadata.Path - } - resolvedEvents[i].GitTargetName = targetMetadata.Name - resolvedEvents[i].GitTargetNamespace = targetMetadata.Namespace - resolvedEvents[i].BootstrapOptions = targetMetadata.BootstrapOptions - } - } - - return &PendingWrite{ - Kind: PendingWriteAtomic, - Events: resolvedEvents, - CommitMessage: request.CommitMessage, - CommitConfig: commitConfig, - Signer: signer, - GitTargetName: request.GitTargetName, - GitTargetNamespace: request.GitTargetNamespace, - Targets: targets, - ByteSize: w.estimateEventsSize(resolvedEvents), - }, nil -} - func (w *BranchWorker) resolveEventsForPendingWrite( ctx context.Context, events []Event, @@ -309,13 +247,13 @@ func (p PendingWrite) Author() string { } // AuthorUserInfo returns the full author identity for commit-shaped pending -// writes, including any OIDC display name and email. Atomic and empty writes -// have no per-user author and return the zero value. +// writes, including any OIDC display name and email. Empty writes have no +// per-user author and return the zero value. func (p PendingWrite) AuthorUserInfo() UserInfo { if p.Kind == PendingWriteRequestRecord { return p.RequestAuthor } - if p.Kind == PendingWriteAtomic || len(p.Events) == 0 { + if len(p.Events) == 0 { return UserInfo{} } return p.Events[0].UserInfo @@ -353,8 +291,7 @@ const ( messageSourceCommitRequestFallback = "commit_request_fallback" // messageSourceLive is a live window rendered through the target's liveTemplate. messageSourceLive = "live" - // messageSourceReconcile is an atomic snapshot or a resync rendered through - // reconcileTemplate. + // messageSourceReconcile is a resync rendered through reconcileTemplate. messageSourceReconcile = "reconcile" // messageSourceUnknown is a write with no message path. It cannot reach the counter today // (such a write fails to render and creates no commit) and exists so that it could never be @@ -364,12 +301,12 @@ const ( // AttributionOutcome returns the attribution outcome for commit-shaped pending writes. It // mirrors AuthorUserInfo: the window is single-author, so the first event's outcome describes -// the whole write. Atomic and empty writes never attempt attribution. +// the whole write. Empty writes never attempt attribution. func (p PendingWrite) AttributionOutcome() AttributionOutcome { if p.Kind == PendingWriteRequestRecord { return p.RequestAttribution } - if p.Kind == PendingWriteAtomic || len(p.Events) == 0 { + if len(p.Events) == 0 { return AttributionNotAttempted } return p.Events[0].Attribution @@ -418,7 +355,7 @@ func (p PendingWrite) Target() ResolvedTargetMetadata { } func (p PendingWrite) targetIdentity() (string, string) { - if p.Kind == PendingWriteAtomic || p.Kind == PendingWriteResync { + if p.Kind == PendingWriteResync { return p.GitTargetName, p.GitTargetNamespace } if len(p.Events) == 0 { diff --git a/internal/git/pending_writes_loop.go b/internal/git/pending_writes_loop.go index fe7d6023b..68cdb6c5a 100644 --- a/internal/git/pending_writes_loop.go +++ b/internal/git/pending_writes_loop.go @@ -14,8 +14,8 @@ import ( // This file is the branch worker's ordered pending writes, and the one driver that turns them into // commits and publications. See docs/design/gittarget-branch-worker-pending-writes.md. // -// Every write the loop makes, whatever produced it (a closed window, an atomic batch, a resync, a -// save's empty record), takes one path: +// Every write the loop makes, whatever produced it (a closed window, a resync, a save's empty +// record), takes one path: // // - decide appends it to the pending writes. The write gates and building the write have passed, so the // decision stands from here on. Deciding does nothing else: no commit, no push, no settlement. @@ -45,12 +45,10 @@ func (e writeFailedError) Error() string { return e.err.Error() } func (e writeFailedError) Unwrap() error { return e.err } func (e writeFailedError) Is(target error) bool { return target == errWriteFailed } -// writeOrigin is what a write's outcome is settled against: the caller waiting on a resync, or the -// atomic request. They are process objects, kept beside -// the write's data rather than in it. +// writeOrigin is what a write's outcome is settled against: the caller waiting on a resync. It is a +// process object, kept beside the write's data rather than in it. type writeOrigin struct { resync *ResyncRequest - atomic *WriteRequest // answered records that the resync's caller was answered already: the remote held it back, // and it is applied later with nobody waiting. answered bool @@ -440,31 +438,22 @@ func (l *branchWorkerEventLoop) settleDropped(pendingWrite PendingWrite, err err } } -// settleFailedChange settles a window or an atomic batch whose commit failed for good. +// settleFailedChange settles a window whose commit failed for good. func (l *branchWorkerEventLoop) settleFailedChange(pendingWrite PendingWrite, err error) { - kind, name, namespace, collection := changeIdentity(pendingWrite) + name, namespace := pendingWrite.windowTarget() + collection := sourceCollectionForEvents(pendingWrite.Events) if isRefusal, refused := l.w.reportPathRefusal(err, name, namespace, collection); isRefusal { - l.w.recordCommitFailure(kind, commitFailureRefused) + l.w.recordCommitFailure(commitFailureRefused) l.reapplyForRefusal(name, namespace, err.Error(), refused, refusalObservationForEvents(pendingWrite.Events, refused), collection) } else { - l.w.recordCommitFailure(kind, commitFailureError) + l.w.recordCommitFailure(commitFailureError) l.w.Log.Error(err, "Commit failed; dropping the write", - "kind", kind, "gitTarget", namespace+"/"+name, "events", len(pendingWrite.Events)) + "gitTarget", namespace+"/"+name, "events", len(pendingWrite.Events)) } l.failDecidedRequest(pendingWrite, fmt.Errorf("commit failed: %w", err)) } -// changeIdentity is a window's or an atomic batch's metric kind, GitTarget, and source collection. -func changeIdentity(pendingWrite PendingWrite) (string, string, string, itypes.CollectionKey) { - if request := pendingWrite.origin.atomic; request != nil { - name, namespace := atomicRefusalTarget(request) - return commitFailureKindAtomic, name, namespace, request.sourceCollection() - } - name, namespace := pendingWrite.windowTarget() - return commitFailureKindWindow, name, namespace, sourceCollectionForEvents(pendingWrite.Events) -} - // replyToDeferredResyncs answers the callers of the resyncs among the writes not committed yet, when // the remote could not be reached or the parent branch a new write branch is created from is // missing. The writes themselves all stay pending for the retry, and admission backpressure bounds @@ -480,7 +469,7 @@ func (l *branchWorkerEventLoop) replyToDeferredResyncs(err error) { req := origin.resync l.w.Log.Error(err, "Cannot refresh the remote before resync; it stays pending", "resources", len(req.Desired), "gitTarget", req.GitTargetNamespace+"/"+req.GitTargetName) - answerResync(req, ResyncResult{Err: fmt.Errorf("refresh remote before resync: %w", err)}) + answerResync(req, ResyncResult{Err: fmt.Errorf("%w: refresh remote before resync: %w", ErrResyncDeferred, err)}) origin.answered = true } } diff --git a/internal/git/pending_writes_test.go b/internal/git/pending_writes_test.go index b12408749..73c10c0a0 100644 --- a/internal/git/pending_writes_test.go +++ b/internal/git/pending_writes_test.go @@ -132,8 +132,8 @@ func TestPendingWriteCommit_DerivesMetadata(t *testing.T) { pendingWrite := PendingWrite{ Kind: PendingWriteCommit, Events: []Event{ - makeEvent("alice", "b"), - makeEvent("alice", "c"), + makeEvent("b"), + makeEvent("c"), }, Targets: map[pendingTargetKey]ResolvedTargetMetadata{ {Name: "team-a", Namespace: "default"}: { @@ -151,33 +151,10 @@ func TestPendingWriteCommit_DerivesMetadata(t *testing.T) { assert.Equal(t, "c", pendingWrite.Events[1].Identifier.Name) } -func TestPendingWriteAtomic_DerivesBatchMetadata(t *testing.T) { +// A resync names its GitTarget on the write itself, not on events, which it carries none of. +func TestPendingWriteResync_TargetIsTheWritesOwn(t *testing.T) { pendingWrite := PendingWrite{ - Kind: PendingWriteAtomic, - Events: []Event{makeEvent("alice", "a"), makeEvent("bob", "b")}, - CommitMessage: "explicit batch message", - GitTargetName: "team-a", - GitTargetNamespace: "default", - Targets: map[pendingTargetKey]ResolvedTargetMetadata{ - {Name: "team-a", Namespace: "default"}: { - Name: "team-a", - Namespace: "default", - Path: "team-team-a", - }, - }, - } - - assert.Empty(t, pendingWrite.Author()) - assert.Equal(t, "explicit batch message", pendingWrite.CommitMessage) - assert.Equal(t, "team-a", pendingWrite.Target().Name) - require.Len(t, pendingWrite.Events, 2) - assert.Equal(t, "a", pendingWrite.Events[0].Identifier.Name) - assert.Equal(t, "b", pendingWrite.Events[1].Identifier.Name) -} - -func TestPendingWriteAtomic_TargetFallsBackToRequestTarget(t *testing.T) { - pendingWrite := PendingWrite{ - Kind: PendingWriteAtomic, + Kind: PendingWriteResync, GitTargetName: "team-a", GitTargetNamespace: "default", } @@ -203,13 +180,13 @@ func TestExecutor_PendingWrites_PreservesArrivalOrder(t *testing.T) { commitsCreated, err := worker.executePendingWrites(context.Background(), repo, []PendingWrite{ { Kind: PendingWriteCommit, - Events: []Event{makeEvent("alice", "a")}, + Events: []Event{makeEvent("a")}, CommitConfig: config, Targets: targets, }, { Kind: PendingWriteCommit, - Events: []Event{makeEvent("alice", "c")}, + Events: []Event{makeEvent("c")}, CommitConfig: config, Targets: targets, }, diff --git a/internal/git/plan_flush.go b/internal/git/plan_flush.go index ae0eb32dc..f9075141c 100644 --- a/internal/git/plan_flush.go +++ b/internal/git/plan_flush.go @@ -761,7 +761,7 @@ func (wb *writeBatch) writeColdBundleMember( rel string, sensitive bool, ) (upsertOutcome, error) { - content, err := wb.writer.buildContentForWrite(ctx, event) + content, err := wb.writer.buildContentForWrite(ctx, wb.target, event) if err != nil { return upsertNoChange, err } @@ -804,7 +804,7 @@ func (wb *writeBatch) coldBundleHasSensitive(rel string) bool { // Append placement for a sensitive resource (see its doc comment), so this path is // plaintext-only. func (wb *writeBatch) appendNewDocument(ctx context.Context, event Event, rel string) (upsertOutcome, error) { - content, err := wb.writer.buildContentForWrite(ctx, event) + content, err := wb.writer.buildContentForWrite(ctx, wb.target, event) if err != nil { return upsertNoChange, err } @@ -1422,7 +1422,7 @@ func gitDocRawObject(content []byte, idx int) (map[string]interface{}, bool) { // file is unsupported, so it is refused), and a write that matches the current bytes is a // no-op (the byte state machine, with the semantic-equality guard for comment-only diffs). func (wb *writeBatch) writeWholeFile(ctx context.Context, event Event, rel string) (upsertOutcome, error) { - content, err := wb.writer.buildContentForWrite(ctx, event) + content, err := wb.writer.buildContentForWrite(ctx, wb.target, event) if err != nil { if wb.writer.isSensitiveIdentifier(event.Identifier) { log.FromContext(ctx).Info( diff --git a/internal/git/plan_flush_test.go b/internal/git/plan_flush_test.go index 1b2e78c0c..dfc394c3f 100644 --- a/internal/git/plan_flush_test.go +++ b/internal/git/plan_flush_test.go @@ -48,7 +48,7 @@ func TestPlanFlush_CreatesNewResourceAtBuiltinPath(t *testing.T) { builtinFull := filepath.Join(root, writer.filePathForIdentifier(event.Identifier)) got, err := os.ReadFile(builtinFull) require.NoError(t, err) - want, err := writer.buildContentForWrite(context.Background(), event) + want, err := writer.buildContentForWrite(context.Background(), placementTarget{}, event) require.NoError(t, err) assert.Equal(t, string(want), string(got), "a new file is the canonical rendering") } diff --git a/internal/git/publication.go b/internal/git/publication.go index b28a431e5..19053f796 100644 --- a/internal/git/publication.go +++ b/internal/git/publication.go @@ -114,7 +114,7 @@ func (w *BranchWorker) recordMaterializationFailure(err error) { reason = "parent_unavailable" } telemetry.GitMaterializationFailuresTotal.Add(context.Background(), 1, - metric.WithAttributes(w.providerAttrs(attribute.String("reason", reason))...)) + metric.WithAttributes(w.gitBranchAttrs(attribute.String("reason", reason))...)) } // setPublicationGaugeSources installs the scrape-time sources of the publication gauges: one @@ -125,7 +125,7 @@ func (m *WorkerManager) setPublicationGaugeSources() { var samples []telemetry.GaugeSample for _, worker := range m.liveWorkers() { if value, ok := read(worker); ok { - samples = append(samples, telemetry.GaugeSample{Value: value, Attrs: worker.providerAttrs()}) + samples = append(samples, telemetry.GaugeSample{Value: value, Attrs: worker.gitBranchAttrs()}) } } return samples diff --git a/internal/git/publication_test.go b/internal/git/publication_test.go index 7f3e07662..be524ae5b 100644 --- a/internal/git/publication_test.go +++ b/internal/git/publication_test.go @@ -85,13 +85,17 @@ func TestPublication_MetricsDescribeTheBacklog(t *testing.T) { telemetry.SetGaugeSource(name, nil) } }) - branch := map[string]string{"branch": "main"} + require.NotEmpty(t, f.worker.GitBranch.Name, "precondition: the fixture's worker writes for a GitBranch") + branch := map[string]string{ + "gitbranch_namespace": f.worker.GitBranch.Namespace, + "gitbranch_name": f.worker.GitBranch.Name, + } gauge := func(name string) (int64, bool) { return telemetry.CollectInt64Sum(reader, "gitopsreverser_"+name, branch) } failures, ok := telemetry.CollectInt64Sum(reader, "gitopsreverser_git_materialization_failures_total", - map[string]string{"branch": "main", "reason": "unreachable"}) + map[string]string{"gitbranch_name": f.worker.GitBranch.Name, "reason": "unreachable"}) require.True(t, ok, "a decided write the remote could not take is counted") assert.Positive(t, failures) diff --git a/internal/git/refresh_test.go b/internal/git/refresh_test.go index f9c7c920c..5d10e2678 100644 --- a/internal/git/refresh_test.go +++ b/internal/git/refresh_test.go @@ -529,10 +529,9 @@ func TestEnqueueRefresh_CountsTheDropOnAFullQueue(t *testing.T) { h.worker.EnqueueRefresh(&RefreshRequest{Target: h.target, Path: h.path, MaxAge: time.Hour}) labels := map[string]string{ - "provider_namespace": h.worker.GitProviderNamespace, - "provider_name": h.worker.GitProviderRef, - "branch": h.worker.Branch, - "kind": queueRefusalRefresh, + "gitbranch_namespace": h.worker.GitBranch.Namespace, + "gitbranch_name": h.worker.GitBranch.Name, + "kind": queueRefusalRefresh, } drops, ok := telemetry.CollectInt64Sum(reader, queueRefusalsMetric, labels) require.True(t, ok, "a dropped refresh must be counted like any other lost work item") @@ -633,8 +632,7 @@ func TestRefresh_ANewBranchDoesNotDependOnARecentRefresh(t *testing.T) { loop := newBranchWorkerEventLoop(h.worker, 0) defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("cm1", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("cm1", "alice", ledgerTargetName)}, }}) require.Empty(t, loop.pendingWrites, "the write was published") diff --git a/internal/git/render_fidelity_drops.go b/internal/git/render_fidelity_drops.go index 5199f3b31..bc7b8c237 100644 --- a/internal/git/render_fidelity_drops.go +++ b/internal/git/render_fidelity_drops.go @@ -17,16 +17,10 @@ import ( // // The gate is per GitTarget and its scopes are per collection: one scope that has not reported // under its current revision closes normal writes for every collection of the target. A live -// event or atomic write that meets a closed gate is DROPPED, and nothing replays it: the watch +// event that meets a closed gate is DROPPED, and nothing replays it: the watch // cursor has already moved past it. So the drop is counted (git_writes_dropped_total) and the first // one under each gate status is logged at Info, naming the scopes the gate waits for. -// The kind label of git_writes_dropped_total. -const ( - writeDroppedEvent = "event" - writeDroppedAtomic = "atomic" -) - // The reason label of git_writes_dropped_total. const ( writeDropRenderFidelityPending = "render_fidelity_pending" @@ -64,21 +58,18 @@ func (l *branchWorkerEventLoop) recordReplayClean(req *ResyncRequest) { // dropAtRenderFidelity counts one write the closed gate dropped, and logs it: at Info for the first // drop of target under this gate status, at V(1) for the rest, so a target whose gate stays closed // through a burst of edits says so once without logging every edit. resource names the dropped -// object, empty for an atomic write. +// object. func (l *branchWorkerEventLoop) dropAtRenderFidelity( target itypes.ResourceReference, status RenderFidelityStatus, - kind, resource string, + resource string, ) { reason := writeDropRenderFidelityPending if status.State == RenderFidelityFalse { reason = writeDropRenderFidelityDiverged } - log := l.w.Log.WithValues("gitTarget", target.String(), "kind", kind, "reason", reason, + log := l.w.Log.WithValues("gitTarget", target.String(), "resource", resource, "reason", reason, "renderFidelity", status.Message) - if resource != "" { - log = log.WithValues("resource", resource) - } mark := renderFidelityDropMark{revision: status.Revision, state: status.State} if l.fidelityDropsLogged[target.Key()] != mark { if l.fidelityDropsLogged == nil { @@ -91,11 +82,11 @@ func (l *branchWorkerEventLoop) dropAtRenderFidelity( } else { log.V(1).Info("Dropped a write because the GitTarget's render fidelity is not established") } - l.recordWriteDropped(kind, reason) + l.recordWriteDropped(reason) } // recordWriteDropped counts one write dropped at the render-fidelity gate. -func (l *branchWorkerEventLoop) recordWriteDropped(kind, reason string) { +func (l *branchWorkerEventLoop) recordWriteDropped(reason string) { if telemetry.GitWritesDroppedTotal == nil { return } @@ -104,5 +95,5 @@ func (l *branchWorkerEventLoop) recordWriteDropped(kind, reason string) { ctx = context.Background() } telemetry.GitWritesDroppedTotal.Add(ctx, 1, metric.WithAttributes( - l.w.providerAttrs(attribute.String("kind", kind), attribute.String("reason", reason))...)) + l.w.gitBranchAttrs(attribute.String("reason", reason))...)) } diff --git a/internal/git/render_fidelity_drops_test.go b/internal/git/render_fidelity_drops_test.go index 4e6132b9e..96c4ba9a1 100644 --- a/internal/git/render_fidelity_drops_test.go +++ b/internal/git/render_fidelity_drops_test.go @@ -67,15 +67,14 @@ func (f *replayFixture) replay() *ResyncRequest { func (f *replayFixture) liveEvent(name string) { f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent(name, "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent(name, "alice", ledgerTargetName)}, }}) } -func writesDropped(t *testing.T, reader *sdkmetric.ManualReader, kind, reason string) int64 { +func writesDropped(t *testing.T, reader *sdkmetric.ManualReader, reason string) int64 { t.Helper() got, _ := telemetry.CollectInt64Sum(reader, "gitopsreverser_git_writes_dropped_total", - map[string]string{"kind": kind, "reason": reason}) + map[string]string{"reason": reason}) return got } @@ -97,7 +96,7 @@ func TestRenderFidelity_ALiveEventQueuedBehindAReplayResyncIsKept(t *testing.T) assert.NotNil(t, f.loop.openWindow, "the event queued behind the replay is kept") assert.True(t, f.gate.AllowsWrites(f.target), "the loop recorded the scope before answering") - assert.Zero(t, writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityPending)) + assert.Zero(t, writesDropped(t, reader, writeDropRenderFidelityPending)) require.NoError(t, (<-req.Result).Err) // The drain records the same result once it has the answer, and the gate takes it unchanged. @@ -153,28 +152,21 @@ func TestRenderFidelity_ADroppedWriteIsCountedAndLoggedOncePerGateStatus(t *test f.liveEvent("first") f.liveEvent("second") - f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("snapshot", "alice", ledgerTargetName)}, - CommitMode: CommitModeAtomic, - }}) assert.Nil(t, f.loop.openWindow, "a pending scope closes the GitTarget") require.NotNil(t, f.loop.held[f.target]) assert.Len(t, f.loop.held[f.target].events, 2, "its live writes are held, not dropped") assert.Len(t, logged.matching("Holding the GitTarget's live writes"), 1, "the hold is said once") - assert.Zero(t, writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityPending)) - assert.Equal(t, int64(1), writesDropped(t, reader, writeDroppedAtomic, writeDropRenderFidelityPending)) - drops := logged.matching("Dropped a write") - require.Len(t, drops, 1) - assert.Contains(t, drops[0], "Waiting for 1 of 1", "it says which scopes the gate waits for") + assert.Zero(t, writesDropped(t, reader, writeDropRenderFidelityPending)) + assert.Empty(t, logged.matching("Dropped a write"), "nothing is dropped while the gate is pending") _, applied := f.gate.RecordScopeDivergence(f.target, f.revision, f.scope.Collection, manifestanalyzer.RenderDivergence{Field: "data.region", Token: "${REGION}"}) require.True(t, applied) f.liveEvent("third") assert.Nil(t, f.loop.held[f.target], "a divergence drops what was held") - assert.Equal(t, int64(3), writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityDiverged)) - assert.Len(t, logged.matching("Dropped a write"), 2, "a new gate status logs again, once") + assert.Equal(t, int64(3), writesDropped(t, reader, writeDropRenderFidelityDiverged)) + assert.Len(t, logged.matching("Dropped a write"), 1, "the drops under one gate status log once") assert.Nil(t, f.loop.openWindow) } diff --git a/internal/git/render_fidelity_gate.go b/internal/git/render_fidelity_gate.go index 270a08f6f..fe1e56ebc 100644 --- a/internal/git/render_fidelity_gate.go +++ b/internal/git/render_fidelity_gate.go @@ -146,7 +146,7 @@ func (g *RenderFidelityGate) Reconcile( // An EMPTY plan restarts every scope vacuously, and a vacuous full restart is the absence // of a measurement rather than a fresh one. Without the length guard a target whose rules // stopped selecting anything would clear a write divergence it never repaired, and the - // writes that do not come from a watch — an atomic request, a CommitRequest — would be + // writes that do not come from a watch — a CommitRequest — would be // admitted again on the strength of nothing. if len(scopes) > 0 && fresh == len(scopes) { state.writeDivergence = nil @@ -256,7 +256,7 @@ func (g *RenderFidelityGate) Status(target types.ResourceReference) RenderFideli return reduceRenderFidelity(state) } -// AllowsWrites reports whether a target may accept a normal live or atomic write. Resync work is +// AllowsWrites reports whether a target may accept a normal live write. Resync work is // deliberately not gated here: it is how the current plan measures and repairs the Git tree. func (g *RenderFidelityGate) AllowsWrites(target types.ResourceReference) bool { return g.Status(target).State == RenderFidelityTrue diff --git a/internal/git/render_fidelity_gate_test.go b/internal/git/render_fidelity_gate_test.go index b226b7b70..3c47ba6d0 100644 --- a/internal/git/render_fidelity_gate_test.go +++ b/internal/git/render_fidelity_gate_test.go @@ -104,12 +104,12 @@ func TestRenderFidelityGate_PendingScopeCannotOpenLiveWindow(t *testing.T) { loop := newBranchWorkerEventLoop(worker, DefaultCommitWindow) event := Event{GitTargetName: target.Name, GitTargetNamespace: target.Namespace, Operation: "UPDATE"} - loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}}}) assert.Nil(t, loop.openWindow, "Unknown must block a new live window") _, applied := gate.RecordScopeClean(target, revisions[scope], scope) require.True(t, applied) - loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}}}) assert.NotNil(t, loop.openWindow, "every scope reporting clean reopens normal writes") } @@ -255,7 +255,7 @@ func TestRenderFidelityGate_DroppedScopeStopsCounting(t *testing.T) { // A plan that selects nothing restarts every scope only vacuously, which is the absence of a // measurement rather than a fresh one. Clearing on it would readmit the writes that do not come -// from a watch — an atomic request, a CommitRequest — on the strength of nothing. +// from a watch — a CommitRequest — on the strength of nothing. func TestRenderFidelityGate_AnEmptyPlanDoesNotClearAWriteDivergence(t *testing.T) { gate := NewRenderFidelityGate() target := types.NewResourceReference("apps", "default") @@ -405,7 +405,7 @@ func TestRenderFidelityGate_InvalidateAdvancesEveryScopeRevision(t *testing.T) { // TestRenderFidelityGate_InvalidateHoldsATargetWithNoScopes is the vacuous case, and it is the one // an unregistered target got wrong: with no scopes to report, an empty plan reduces to True, so a // target whose rules select nothing would have been writable against a repository nothing has -// looked at. Atomic requests and CommitRequests do not come from a watch. +// looked at. A CommitRequest does not come from a watch. func TestRenderFidelityGate_InvalidateHoldsATargetWithNoScopes(t *testing.T) { gate := NewRenderFidelityGate() target := types.NewResourceReference("apps", "default") diff --git a/internal/git/render_fidelity_hold.go b/internal/git/render_fidelity_hold.go index 69afd2727..496561564 100644 --- a/internal/git/render_fidelity_hold.go +++ b/internal/git/render_fidelity_hold.go @@ -127,7 +127,7 @@ func (l *branchWorkerEventLoop) serviceHeld(target itypes.ResourceReference) { case status.State == RenderFidelityFalse: l.takeHeld(target) for _, h := range held.events { - l.dropAtRenderFidelity(target, status, writeDroppedEvent, h.event.Identifier.String()) + l.dropAtRenderFidelity(target, status, h.event.Identifier.String()) } case l.w.now().Sub(held.since) >= renderFidelityHoldLimit: l.giveUpHold(target, "the GitTarget's render fidelity has been pending for "+ @@ -158,7 +158,7 @@ func (l *branchWorkerEventLoop) giveUpHold(target itypes.ResourceReference, why return } for range held.events { - l.recordWriteDropped(writeDroppedEvent, writeDropRenderFidelityPending) + l.recordWriteDropped(writeDropRenderFidelityPending) } l.w.Log.Info("Dropped the live writes held for a GitTarget, and asked for a fresh snapshot of it, which "+ "commits the current state as the operator. A deletion comes back only under prune mode Always", diff --git a/internal/git/render_fidelity_hold_test.go b/internal/git/render_fidelity_hold_test.go index 8f169fa4b..aca6f7e9e 100644 --- a/internal/git/render_fidelity_hold_test.go +++ b/internal/git/render_fidelity_hold_test.go @@ -31,7 +31,7 @@ func (f *replayFixture) streamEvent(name, author string, operation itypes.Operat } func (f *replayFixture) send(events ...Event) { - f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: events, CommitMode: CommitModePerEvent}}) + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: events}}) } // windowNames is the objects the open window holds, in the order they were added. @@ -69,7 +69,7 @@ func TestRenderFidelityHold_AnEditMadeWhileAnotherCollectionReplaysIsCommittedBy assert.Equal(t, "alice", f.loop.openWindow.Author, "with its author") assert.Empty(t, f.loop.held) assert.Zero(t, f.loop.heldEventBytes) - assert.Zero(t, writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityPending)) + assert.Zero(t, writesDropped(t, reader, writeDropRenderFidelityPending)) } // TestRenderFidelityHold_HeldEditsGoBeforeLaterOnes: a gate can open between two wakes, and the next @@ -321,7 +321,7 @@ func TestRenderFidelityHold_AHoldPastItsLimitAsksForAFreshSnapshot(t *testing.T) assert.Empty(t, f.loop.held, "given up") assert.Zero(t, f.loop.heldEventBytes) assert.Equal(t, []itypes.ResourceReference{f.target}, asked, "a fresh snapshot is asked for") - assert.Equal(t, int64(2), writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityPending)) + assert.Equal(t, int64(2), writesDropped(t, reader, writeDropRenderFidelityPending)) gaveUp := logged.matching("Dropped the live writes held") require.Len(t, gaveUp, 1) assert.Contains(t, gaveUp[0], `"deletions"=1`, "it says how many deletions a snapshot may not bring back") @@ -359,5 +359,5 @@ func TestRenderFidelityHold_ADivergenceAfterAReplayDropsWhatWasHeld(t *testing.T assert.Empty(t, f.loop.held) assert.Nil(t, f.loop.openWindow) - assert.Equal(t, int64(1), writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityDiverged)) + assert.Equal(t, int64(1), writesDropped(t, reader, writeDropRenderFidelityDiverged)) } diff --git a/internal/git/replay_failure_test.go b/internal/git/replay_failure_test.go index 6ade57ec5..b11d2d264 100644 --- a/internal/git/replay_failure_test.go +++ b/internal/git/replay_failure_test.go @@ -78,8 +78,7 @@ func TestReplayFailure_DoesNotLetThePushSettleWorkThatWasReset(t *testing.T) { // One retained write, committed locally and waiting out the push cooldown. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("will-be-lost", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("will-be-lost", "alice", ledgerTargetName)}, }}) require.True(t, loop.finalizeOpenWindow()) require.Len(t, loop.pendingWrites, 1) @@ -127,8 +126,7 @@ func TestReplayFailure_DoesNotResolveACommitRequestThatNeverLanded(t *testing.T) defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("saved-by-hand", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("saved-by-hand", "alice", ledgerTargetName)}, }}) require.NotNil(t, loop.openWindow) req := attachReq("alice", 0) @@ -191,8 +189,7 @@ func TestReplayFailure_HoldsAWholeBatchNotJustItsFirstWrite(t *testing.T) { // Two retained writes, each its own local commit. for _, name := range []string{"first-of-two", "second-of-two"} { loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent(name, "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent(name, "alice", ledgerTargetName)}, }}) require.True(t, loop.finalizeOpenWindow()) } @@ -228,8 +225,7 @@ func TestReplayFailure_RecoveryRunsBeforeEveryCommitPathToo(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("retained-across-replay", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("retained-across-replay", "alice", ledgerTargetName)}, }}) require.True(t, loop.finalizeOpenWindow()) @@ -241,8 +237,7 @@ func TestReplayFailure_RecoveryRunsBeforeEveryCommitPathToo(t *testing.T) { // A second write arrives. Finalizing it recovers first, so both writes are real commits again. loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("arrived-after-replay", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("arrived-after-replay", "alice", ledgerTargetName)}, }}) require.True(t, loop.finalizeOpenWindow()) assert.True(t, loop.checkoutCurrent(), "the commit path rebuilt the stranded write") @@ -270,8 +265,7 @@ func TestReplayFailure_OnTheContentionPathIsAlsoHeld(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("lost-to-contention", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("lost-to-contention", "alice", ledgerTargetName)}, }}) require.True(t, loop.finalizeOpenWindow()) require.Len(t, loop.pendingWrites, 1) @@ -323,8 +317,7 @@ func TestReplayFailure_AResetThatDiesHalfwayIsAlsoHeld(t *testing.T) { defer loop.stopTimers() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("must-survive", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("must-survive", "alice", ledgerTargetName)}, }}) require.True(t, loop.finalizeOpenWindow()) require.Len(t, loop.pendingWrites, 1) diff --git a/internal/git/replay_refusal_test.go b/internal/git/replay_refusal_test.go index e3deddf26..7a175e801 100644 --- a/internal/git/replay_refusal_test.go +++ b/internal/git/replay_refusal_test.go @@ -51,8 +51,7 @@ func newReplayRefusalFixture(t *testing.T, slug string) (*ledgerFixture, *branch func retainTwoWrites(t *testing.T, loop *branchWorkerEventLoop) { t.Helper() loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("refused-on-replay", "alice", ledgerTargetName)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("refused-on-replay", "alice", ledgerTargetName)}, }}) require.NotNil(t, loop.openWindow) req := attachReq("alice", 0) @@ -63,8 +62,7 @@ func retainTwoWrites(t *testing.T, loop *branchWorkerEventLoop) { require.NotNil(t, loop.pendingWrites[0].CommitRequest) loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("kept-behind-it", "bob", replayRefusalTargetB)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("kept-behind-it", "bob", replayRefusalTargetB)}, }}) require.True(t, loop.finalizeOpenWindow()) require.Len(t, loop.pendingWrites, 2) @@ -173,8 +171,7 @@ func TestReplayRefusal_AResyncAnsweredEarlierIsReportedNotAnsweredAgain(t *testi require.Len(t, loop.pendingWrites, 1, "the resync is retained as a local commit") loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("kept-behind-it", "bob", replayRefusalTargetB)}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("kept-behind-it", "bob", replayRefusalTargetB)}, }}) require.True(t, loop.finalizeOpenWindow()) require.Len(t, loop.pendingWrites, 2) diff --git a/internal/git/repository_init_fixture_test.go b/internal/git/repository_init_fixture_test.go new file mode 100644 index 000000000..eacf21058 --- /dev/null +++ b/internal/git/repository_init_fixture_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "fmt" + + gogit "github.com/go-git/go-git/v6" +) + +// A test fixture, not a production path: it clones the worker's repository the way a first +// publication does, without counting a fetch, so a test can start from a prepared checkout and +// measure only what it is about. + +// ensureRepositoryInitialized ensures the worker's repository is cloned and ready. +func (w *BranchWorker) ensureRepositoryInitialized(ctx context.Context) error { + provider, err := w.getGitProvider(ctx) + if err != nil { + return fmt.Errorf("failed to get GitProvider: %w", err) + } + + repoPath := w.repoPath() + if w.shouldReadRetainedLocalRepository(repoPath) { + return nil + } + + auth, err := getAuthFromSecret(ctx, w.Client, provider, w.sshHostKeys, w.credentialPolicy) + if err != nil { + return fmt.Errorf("failed to get auth: %w", err) + } + + // Use new PrepareBranch abstraction + pullReport, err := w.prepareBranch(ctx, repoPath, auth) + if err != nil { + return fmt.Errorf("failed to prepare repository: %w", err) + } + + // Update metadata from pull report + w.updateBranchMetadataFromPullReport(pullReport) + + return nil +} + +func (w *BranchWorker) shouldReadRetainedLocalRepository(repoPath string) bool { + if !w.hasUnpushedWork.Load() && w.inflightItems.Load() == 0 { + return false + } + + if _, err := gogit.PlainOpen(repoPath); err != nil { + return false + } + + return true +} diff --git a/internal/git/request_template_test.go b/internal/git/request_template_test.go index 5cb6f43b9..41d4cfd64 100644 --- a/internal/git/request_template_test.go +++ b/internal/git/request_template_test.go @@ -245,25 +245,10 @@ func TestNewRequestTemplateProbe_IsFreshEachTime(t *testing.T) { } // requestTemplate describes the commit a CommitRequest's message produces, and a CommitRequest -// attaches to a live window — which finalizes as PendingWriteCommit. An atomic snapshot can also -// carry a message, and it must keep the behaviour it had before requestTemplate existed: committed -// verbatim, never framed with a live context it was not written for. -func TestResolveMessage_AtomicWriteIsNeverFramed(t *testing.T) { - config := requestConfig(requestBodyTemplate) - - atomic := requestWrite("chore: snapshot", config) - atomic.Kind = PendingWriteAtomic - assert.Equal(t, messageResolutionRequest, atomic.resolveMessage(), - "an atomic write with a message stays verbatim even when requestTemplate is configured") - assert.Equal(t, messageSourceCommitRequest, atomic.messageSource()) - - // And with no message it still renders reconcileTemplate, as documented. - plainAtomic := requestWrite("", config) - plainAtomic.Kind = PendingWriteAtomic - assert.Equal(t, messageResolutionReconcileTemplate, plainAtomic.resolveMessage()) - - // A resync is untouched either way. - resync := requestWrite("pre-rendered", config) +// attaches to a live window, which finalizes as PendingWriteCommit. A resync arrives pre-rendered +// and is never framed, whatever requestTemplate says. +func TestResolveMessage_AResyncIsNeverFramed(t *testing.T) { + resync := requestWrite("pre-rendered", requestConfig(requestBodyTemplate)) resync.Kind = PendingWriteResync assert.Equal(t, messageResolutionPreRendered, resync.resolveMessage()) } diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index 356f5258d..bb553ba68 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -22,10 +22,10 @@ import ( itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) -// applyResync decides one revision-pinned resync in order on the worker goroutine. It mirrors the -// atomic-commit path: any open live window is finalized first so arrival order is preserved; the -// resync is then decided like every other write, and its caller answered when its commit is made -// or fails (settleResyncApplied, settleFailed), or as soon as the remote holds it back +// applyResync decides one revision-pinned resync in order on the worker goroutine. Any open live +// window is finalized first so arrival order is preserved; the resync is then decided like every +// other write, and its caller answered when its commit is made or fails (settleResyncApplied, +// settleFailed), or as soon as the remote holds it back // (replyToDeferredResyncs). A resync that commits nothing stops being pending at once, so it never // schedules a push. A build or commit failure answers with the error and commits nothing: the // gatherer already guaranteed the snapshot is complete, so a failure here is a write fault, never a diff --git a/internal/git/resync_push_test.go b/internal/git/resync_push_test.go index 4087d923c..03e24b21b 100644 --- a/internal/git/resync_push_test.go +++ b/internal/git/resync_push_test.go @@ -326,8 +326,7 @@ func TestHandleResyncRequest_ClosedWindowIsPushedEvenWhenNoOpResync(t *testing.T // A live ConfigMap edit opens a window (commitWindow is an hour, so it stays open). loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, - CommitMode: CommitModePerEvent, + Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, }}) require.NotNil(t, loop.openWindow, "the edit must open a window") diff --git a/internal/git/source_collection.go b/internal/git/source_collection.go index ddff38150..912dbd274 100644 --- a/internal/git/source_collection.go +++ b/internal/git/source_collection.go @@ -48,10 +48,8 @@ func sourceCollectionForLog(c types.CollectionKey) string { return c.String() } -// sourceCollection is the write request's source collection: the one its events agree on, or the zero collection -// when they do not. The live path wraps exactly one event, which is the path that carries a -// collection; an atomic reconcile carries many events and speaks for no single collection, so it reads -// unclaimed, which is what it is. +// sourceCollection is the write request's source collection: the one its events agree on, or the +// zero collection when they do not. The live path wraps exactly one event. func (r *WriteRequest) sourceCollection() types.CollectionKey { if r == nil || len(r.Events) == 0 { return types.CollectionKey{} diff --git a/internal/git/testdata/git-roundtrip-ledger.golden b/internal/git/testdata/git-roundtrip-ledger.golden index 5319eb667..9f0f7745f 100644 --- a/internal/git/testdata/git-roundtrip-ledger.golden +++ b/internal/git/testdata/git-roundtrip-ledger.golden @@ -18,8 +18,8 @@ operation conn fetch-open fetch-xfer push-open push-xfer ------------------------------------------------------------------------------------------------------------------------------ -1. bootstrap helper, empty remote 1 1 -/<1K 0 0 0 -2. bootstrap helper, populated remote 3 2 -/<1K 1 <1K/<1K 0 0 +1. first clone, empty remote 1 1 -/<1K 0 0 0 +2. first clone, populated remote 3 2 -/<1K 1 <1K/<1K 0 0 3. one publication, one commit 2 0 0 1 -/<1K 1 <1K/<1K 4. one publication, several commits in one cycle 2 0 0 1 -/<1K 1 <4K/<1K 5. a publication that commits nothing 1 0 0 1 -/<1K 0 diff --git a/internal/git/types.go b/internal/git/types.go index d70ad18e7..47059fa83 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -247,28 +247,18 @@ type UserInfo struct { Email string } -// CommitMode defines how a write request should be committed. -type CommitMode string - -const ( - // CommitModePerEvent streams request events through the live commit window. - // With commitWindow=0 each event finalizes immediately; otherwise events - // coalesce by author and target until the window's idle timeout or maximum duration. - CommitModePerEvent CommitMode = "per_event" - // CommitModeAtomic creates one commit for all events in the request. - CommitModeAtomic CommitMode = "atomic" -) - -// WriteRequest is the unit of work queued and written by the BranchWorker. +// WriteRequest is the unit of work queued and written by the BranchWorker: live events, which +// stream through the commit window. Each event names its own GitTarget. type WriteRequest struct { - Events []Event - CommitMessage string - CommitConfig *CommitConfig - Signer gogit.Signer - GitTargetName string - GitTargetNamespace string - BootstrapOptions pathBootstrapOptions - CommitMode CommitMode + Events []Event +} + +// gitTarget is the GitTarget the request's first event writes for, as namespace/name, for logs. +func (r *WriteRequest) gitTarget() string { + if r == nil || len(r.Events) == 0 { + return "" + } + return r.Events[0].GitTargetNamespace + "/" + r.Events[0].GitTargetName } // PendingWriteKind distinguishes the durable write shapes retained until push. @@ -277,9 +267,6 @@ type PendingWriteKind string const ( // PendingWriteCommit is one finalized commit-shaped live-event window. PendingWriteCommit PendingWriteKind = "grouped_window" - // PendingWriteAtomic is a caller-defined atomic request, typically from - // reconciliation. - PendingWriteAtomic PendingWriteKind = "atomic" // PendingWriteResync is a streaming-snapshot resync (M8): it carries the COMPLETE // desired resource set for one GitTarget, and the worker materialises it with a // content-derived mark-and-sweep against the worktree (upsert every desired diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index 5b5d1cdb4..ab7f066ac 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -887,7 +887,7 @@ func (m *WorkerManager) queueDepthSamples() []telemetry.GaugeSample { for _, worker := range workers { samples = append(samples, telemetry.GaugeSample{ Value: worker.queueDepth(), - Attrs: worker.providerAttrs(), + Attrs: worker.gitBranchAttrs(), }) } return samples diff --git a/internal/git/write_boundary_precondition_test.go b/internal/git/write_boundary_precondition_test.go index 201c7982d..99a5e51ff 100644 --- a/internal/git/write_boundary_precondition_test.go +++ b/internal/git/write_boundary_precondition_test.go @@ -246,7 +246,7 @@ func TestEventLoop_LiveFanInRefusal_FailsGitTargetAndCommitsNothing(t *testing.T event.UserInfo = UserInfo{Username: "alice"} event.GitTargetName = "podinfo-test" event.GitTargetNamespace = "default" - loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}}}) require.NotNil(t, loop.openWindow, "the live event must open a commit window") assert.False(t, loop.finalizeOpenWindow(), "a refused write plan must not produce a pending write") diff --git a/internal/giteaclient/keys.go b/internal/giteaclient/keys.go index e007acc32..39454e6d1 100644 --- a/internal/giteaclient/keys.go +++ b/internal/giteaclient/keys.go @@ -162,30 +162,6 @@ func (c *Client) EnsureUserKeyAsAdmin(ctx context.Context, login, title, publicK return c.RegisterUserKeyAsAdmin(ctx, login, title, publicKey) } -// RegisterUserKeyAsUser registers a public key via POST /user/keys using the -// currently authenticated user. Use this when comparing whether admin-upload -// vs user-upload lands the key with a different key_type in Gitea's classification. -func (c *Client) RegisterUserKeyAsUser(ctx context.Context, title, publicKey string) (*PublicKey, error) { - publicKey = strings.TrimSpace(publicKey) - if publicKey == "" { - return nil, errors.New("public key is empty") - } - payload := map[string]any{"title": title, "key": publicKey, "read_only": false} - var created PublicKey - code, raw, err := c.Do(ctx, http.MethodPost, "/user/keys", payload, &created) - if err != nil { - return nil, err - } - switch code { - case http.StatusCreated: - return &created, nil - case http.StatusUnprocessableEntity: - return nil, fmt.Errorf("register user key: HTTP 422: %s", TruncateBody(string(raw))) - default: - return nil, unexpectedStatus(http.MethodPost, "/user/keys", code, raw) - } -} - // GetVerificationToken returns the per-user verification token. The same token // is used by the GPG and SSH verification flows; Gitea's web form accepts an // SSH signature over it (namespace "gitea") to flip public_key.verified=1. diff --git a/internal/telemetry/apply_requests.go b/internal/telemetry/apply_requests.go index a759cbe0d..cfe322bbf 100644 --- a/internal/telemetry/apply_requests.go +++ b/internal/telemetry/apply_requests.go @@ -5,7 +5,8 @@ package telemetry import "go.opentelemetry.io/otel/metric" // ApplyRequestsTotal counts the apply attempts GitOps Reverser made, one per attempt to one applier -// object, labelled by {applier_kind, reason, outcome}. +// object, labelled by {gitbranch_namespace, gitbranch_name, provider_namespace, provider_name, +// applier_kind, claim_namespace, claim_name, reason, outcome}. // // applier_kind names the applier: "kustomization" (a Flux Kustomization) or "application" (an Argo // CD Application). reason is the attempt's ApplyReason (ForeignCommit, Refusal, Repair, CatchUp), @@ -31,6 +32,9 @@ import "go.opentelemetry.io/otel/metric" // followed by the branch worker, and an ApplyFailed or ApplyUnresolved Event marks one that did not // complete, or not in time (apply_operations_unresolved counts the latter). // +// The GitBranch, with its GitProvider as every branch-worker series carries it, is the worker that +// decided the attempt, and the claim is the one holding the applier +// it asked, so both are owners rather than guesses, and "which applier keeps failing" is a selector. // No GitTarget identity: one attempt is for one applier, which several targets can share, so a // target label would have to invent an owner. The ApplyRequested and ApplyRequestFailed Events on // each GitTarget carry that identity instead. @@ -48,7 +52,7 @@ var ApplyRequestsTotal metric.Int64Counter var ClaimedResourceReconciledTotal metric.Int64Counter // ApplyRevisionMismatchesTotal counts apply operations that succeeded on another revision than the -// commit GitOps Reverser checked, labelled by {applier_kind}. A Flux Kustomization applies whatever +// commit GitOps Reverser checked, labelled as ApplyRequestsTotal is, without reason and outcome. A Flux Kustomization applies whatever // its GitRepository's artifact holds when it reconciles; the artifact is read again just before the // resume, so this is somebody stamping the GitRepository between that read and the reconcile (a // person, or the source's own interval). The apply window still withholds a late write over a diff --git a/internal/telemetry/apply_window.go b/internal/telemetry/apply_window.go index 08d7b2f15..65c3c2e73 100644 --- a/internal/telemetry/apply_window.go +++ b/internal/telemetry/apply_window.go @@ -6,8 +6,8 @@ import "go.opentelemetry.io/otel/metric" // ApplyWindowCollisionsTotal counts writes by an applier, inside its apply window (from the attempt // being sent until its operation completed, plus a short grace), to an object the API side edited after the -// attempt was sent: the apply arriving late over a newer edit. Labelled by {provider_namespace, -// provider_name, branch, outcome}. outcome is a frozen enum: +// attempt was sent: the apply arriving late over a newer edit. Labelled by {gitbranch_*, +// provider_*, outcome}. outcome is a frozen enum: // // withheld — the write is proven to be the applier's (its field manager alone wrote last), so it // was not mirrored. Git keeps the API's value, and a Repair is pending for the applier, an @@ -23,7 +23,7 @@ import "go.opentelemetry.io/otel/metric" var ApplyWindowCollisionsTotal metric.Int64Counter // ApplyEchoesTotal counts the applier's own writes the apply window judged: an apply GitOps Reverser -// asked for landing, object by object. Labelled by {provider_namespace, provider_name, branch, +// asked for landing, object by object. Labelled by {gitbranch_*, provider_*, // outcome}. outcome is a frozen enum: // // not_mirrored — the applier's managers alone wrote the object last, at or after the earliest send @@ -39,8 +39,8 @@ var ApplyEchoesTotal metric.Int64Counter // ApplyOwnershipConflictsTotal counts objects another applier owns that a branch worker kept out of a // coordinated folder: a live write not mirrored, or a snapshot object neither upserted -// nor swept (docs/design/applier-echoes.md, F4). Labelled by {provider_namespace, provider_name, -// branch}. Any non-zero rate is a misconfiguration: another Kustomization or Application applies +// nor swept (docs/design/applier-echoes.md, F4). Labelled by {gitbranch_*, +// provider_*}. Any non-zero rate is a misconfiguration: another Kustomization or Application applies // objects into a namespace the folder mirrors, and its log line names both. // // Registered in registerCounters (exporter.go) with every other counter. @@ -48,8 +48,8 @@ var ApplyOwnershipConflictsTotal metric.Int64Counter // StaleSnapshotsTotal counts resync snapshots a branch worker found stale: collected while Git was // ahead of the cluster, across an apply, or arriving while Git is ahead, so an apply was about to make -// the picture they hold wrong (docs/design/applier-echoes.md, F3). Labelled by {provider_namespace, -// provider_name, branch, outcome}. outcome is a frozen enum: +// the picture they hold wrong (docs/design/applier-echoes.md, F3). Labelled by {gitbranch_*, +// provider_*, outcome}. outcome is a frozen enum: // // discarded — the snapshot was not decided, and a fresh one of its GitTarget is asked for once Git // is no longer ahead. diff --git a/internal/telemetry/branch_targets.go b/internal/telemetry/branch_targets.go index f094162a9..b622f3dab 100644 --- a/internal/telemetry/branch_targets.go +++ b/internal/telemetry/branch_targets.go @@ -2,11 +2,7 @@ package telemetry -import ( - "sync" - - "go.opentelemetry.io/otel/attribute" -) +import "go.opentelemetry.io/otel/attribute" // git_branch_targets is the join between the two halves of the pipeline, and it exists because // they do not share a label set. @@ -14,9 +10,9 @@ import ( // Everything up to the branch worker is labelled by the GitTarget that asked for the write — // watch_events_total, git_documents_total, watch_recovery_total, the placement counters — because // "which tenant stopped receiving events" is the question those answer. Everything from the branch -// worker on is labelled {provider_namespace, provider_name, branch}, because a BranchWorker serves -// a (GitProvider, branch) and several GitTargets can share one while writing different paths. So -// git_pushes_total{outcome="failed"} names a branch that stopped advancing and CANNOT name the +// worker on is labelled {gitbranch_namespace, gitbranch_name}, because a BranchWorker serves one +// GitBranch and several GitTargets can share one while writing different paths. So +// git_pushes_total{outcome="failed"} names a GitBranch that stopped advancing and CANNOT name the // GitTargets now behind it, which is the question an operator asks during the incident. // // The fix is a join series rather than a label. Putting gittarget_* on git_commits_total would be @@ -24,13 +20,16 @@ import ( // would have to be invented. This publishes the mapping instead, always 1, bounded by GitTarget // count, and PromQL does the join — the same shape attribution_transport_info already uses. // +// The mapping is the GitTarget's spec.gitBranchRef, its namespace resolved, and nothing else: it +// holds whether or not that GitBranch exists, holds the branch, or has a running worker. +// // # What it does NOT assert // -// It says which GitTargets write to a branch. It does not say any of them had pending work, so a -// failing branch names POTENTIALLY affected targets and never "targets that are behind". Two -// consequences worth knowing before writing an alert on it: +// It says which GitTargets write through a GitBranch. It does not say any of them had pending +// work, so a failing GitBranch names POTENTIALLY affected targets and never "targets that are +// behind". Two consequences worth knowing before writing an alert on it: // -// - A branch with no push series at all drops out of a multiplication entirely, which is the +// - A GitBranch with no push series at all drops out of a multiplication entirely, which is the // silent case: the target an operator most wants to see is the one whose worker never ran. // Missing workers need their own query against this mapping and git_queue_depth. // - spec.suspend stops the WRITE and nothing else, so a suspended target keeps its worker, its @@ -59,76 +58,27 @@ import ( // never got a worker still appears. Publishing from the live worker map would hide exactly the // case the join is for. See docs/interpreting-metrics.md for the two queries this enables. -// branchTargetKey identifies the GitTarget one published series is about. One GitTarget has one -// destination — spec.branch, spec.path and spec.gitProviderRef are all CEL-immutable — so the key -// is the target and the destination is the value. -type branchTargetKey struct { - namespace string - name string -} - -// branchTargetDestination is where that GitTarget writes, as the Git-side instruments label it, -// plus the cluster it mirrors FROM. -type branchTargetDestination struct { - providerNamespace string - providerName string - branch string - sourceCluster string -} - -var ( - branchTargetsMu sync.RWMutex - branchTargets = map[branchTargetKey]branchTargetDestination{} -) +// branchTargets holds one series per GitTarget. One GitTarget has one destination — +// spec.gitBranchRef and spec.path are both CEL-immutable — so the key is the target and the +// GitBranch is the value. +var branchTargets = newObjectSeries("gittarget_namespace", "gittarget_name") -// RecordBranchTarget publishes one GitTarget's destination, replacing any previous entry for it. +// RecordBranchTarget publishes one GitTarget's GitBranch, and the cluster it mirrors FROM, +// replacing any previous entry for it. // -// Call it on every reconcile that resolves a destination, including one that then fails a gate: a -// gauge is level rather than event, and the join has to carry the targets that are not working. -func RecordBranchTarget(namespace, name, providerNamespace, providerName, branch, sourceCluster string) { - branchTargetsMu.Lock() - defer branchTargetsMu.Unlock() - branchTargets[branchTargetKey{namespace: namespace, name: name}] = branchTargetDestination{ - providerNamespace: providerNamespace, - providerName: providerName, - branch: branch, - sourceCluster: sourceCluster, - } +// Call it on every reconcile, including one that then fails a gate: a gauge is level rather than +// event, and the join has to carry the targets that are not working. +func RecordBranchTarget(namespace, name, gitBranchNamespace, gitBranchName, sourceCluster string) { + branchTargets.record(namespace, name, + attribute.String("gitbranch_namespace", gitBranchNamespace), + attribute.String("gitbranch_name", gitBranchName), + attribute.String("source_cluster", sourceCluster), + ) } // ForgetBranchTarget stops publishing the series for one GitTarget. It is the delete path, and it // is the half that has to be right: a join series that outlives its GitTarget keeps attributing a // live branch's failures to an object that no longer exists. func ForgetBranchTarget(namespace, name string) { - branchTargetsMu.Lock() - defer branchTargetsMu.Unlock() - delete(branchTargets, branchTargetKey{namespace: namespace, name: name}) -} - -// branchTargetSamples renders the mapping as gauge samples, one per GitTarget, read at scrape time. -// -// Observable rather than pushed for the reason resource_condition is: this family needs series that -// STOP. A pushed gauge latches its last value, so a deleted GitTarget would stay in the join for -// the life of the process. Reading a map at scrape time makes the delete a map delete. -func branchTargetSamples() []GaugeSample { - branchTargetsMu.RLock() - defer branchTargetsMu.RUnlock() - - samples := make([]GaugeSample, 0, len(branchTargets)) - for key, dest := range branchTargets { - samples = append(samples, GaugeSample{ - Value: 1, - Attrs: []attribute.KeyValue{ - // Prefixed, never a bare namespace/name: a Prometheus pod scrape with - // honor_labels=false overwrites those with the scraped pod's own. - attribute.String("gittarget_namespace", key.namespace), - attribute.String("gittarget_name", key.name), - attribute.String("provider_namespace", dest.providerNamespace), - attribute.String("provider_name", dest.providerName), - attribute.String("branch", dest.branch), - attribute.String("source_cluster", dest.sourceCluster), - }, - }) - } - return samples + branchTargets.forget(namespace, name) } diff --git a/internal/telemetry/branch_targets_test.go b/internal/telemetry/branch_targets_test.go index 38b5ded3a..68aa5eac7 100644 --- a/internal/telemetry/branch_targets_test.go +++ b/internal/telemetry/branch_targets_test.go @@ -26,17 +26,17 @@ func branchTargetSeries( }) } -// The whole point of the instrument: two GitTargets sharing one provider and branch each publish -// their own series, so a query keyed on {provider_namespace, provider_name, branch} fans out to -// both. One series per branch would answer the easy case and lose the case the ask was filed about. +// The whole point of the instrument: two GitTargets sharing one GitBranch each publish their own +// series, so a query keyed on {gitbranch_namespace, gitbranch_name} fans out to both. One series +// per branch would answer the easy case and lose the case the ask was filed about. func TestBranchTargets_TwoTargetsOnOneBranchEachPublish(t *testing.T) { reader, err := InitTestExporter() require.NoError(t, err) defer ForgetBranchTarget("team-a", "mirror") defer ForgetBranchTarget("team-a", "docs") - RecordBranchTarget("team-a", "mirror", "team-a", "acme", "main", "default") - RecordBranchTarget("team-a", "docs", "team-a", "acme", "main", "default") + RecordBranchTarget("team-a", "mirror", "team-a", "acme-main", "default") + RecordBranchTarget("team-a", "docs", "team-a", "acme-main", "default") for _, name := range []string{"mirror", "docs"} { value, ok := branchTargetSeries(t, reader, "team-a", name) @@ -47,13 +47,12 @@ func TestBranchTargets_TwoTargetsOnOneBranchEachPublish(t *testing.T) { // The join keys have to be exactly the ones the Git-side instruments carry, or the vector // match silently returns nothing. value, ok := CollectInt64Sum(reader, branchTargetsMetric, map[string]string{ - "provider_namespace": "team-a", - "provider_name": "acme", - "branch": "main", + "gitbranch_namespace": "team-a", + "gitbranch_name": "acme-main", "gittarget_namespace": "team-a", "gittarget_name": "mirror", }) - require.True(t, ok, "the destination labels must match providerAttrs' key names exactly") + require.True(t, ok, "the destination labels must match gitBranchAttrs' key names exactly") assert.Equal(t, int64(1), value) } @@ -63,7 +62,7 @@ func TestBranchTargets_ForgetStopsTheSeries(t *testing.T) { reader, err := InitTestExporter() require.NoError(t, err) - RecordBranchTarget("team-a", "mirror", "team-a", "acme", "main", "default") + RecordBranchTarget("team-a", "mirror", "team-a", "acme-main", "default") _, ok := branchTargetSeries(t, reader, "team-a", "mirror") require.True(t, ok) @@ -79,8 +78,8 @@ func TestBranchTargets_ForgetIsScopedToOneTarget(t *testing.T) { require.NoError(t, err) defer ForgetBranchTarget("team-a", "docs") - RecordBranchTarget("team-a", "mirror", "team-a", "acme", "main", "default") - RecordBranchTarget("team-a", "docs", "team-a", "acme", "main", "default") + RecordBranchTarget("team-a", "mirror", "team-a", "acme-main", "default") + RecordBranchTarget("team-a", "docs", "team-a", "acme-main", "default") ForgetBranchTarget("team-a", "mirror") @@ -98,7 +97,7 @@ func TestBranchTargets_RepeatedRecordStaysOneSeries(t *testing.T) { defer ForgetBranchTarget("team-a", "mirror") for range 5 { - RecordBranchTarget("team-a", "mirror", "team-a", "acme", "main", "default") + RecordBranchTarget("team-a", "mirror", "team-a", "acme-main", "default") } value, ok := branchTargetSeries(t, reader, "team-a", "mirror") @@ -115,8 +114,8 @@ func TestBranchTargets_SourceClusterDistinguishesLocalFromRemote(t *testing.T) { defer ForgetBranchTarget("team-a", "local") defer ForgetBranchTarget("team-a", "remote") - RecordBranchTarget("team-a", "local", "team-a", "acme", "main", "gitops-reverser/local") - RecordBranchTarget("team-a", "remote", "team-a", "acme", "main", "team-a/tenant-eu-1") + RecordBranchTarget("team-a", "local", "team-a", "acme-main", "gitops-reverser/local") + RecordBranchTarget("team-a", "remote", "team-a", "acme-main", "team-a/tenant-eu-1") for name, cluster := range map[string]string{"local": "gitops-reverser/local", "remote": "team-a/tenant-eu-1"} { value, ok := CollectInt64Sum(reader, branchTargetsMetric, map[string]string{ @@ -141,15 +140,14 @@ func TestBranchTargets_OneBranchCanSpanTwoClusters(t *testing.T) { defer ForgetBranchTarget("team-a", "from-eu") defer ForgetBranchTarget("team-a", "from-us") - RecordBranchTarget("team-a", "from-eu", "team-a", "acme", "main", "tenant-eu-1") - RecordBranchTarget("team-a", "from-us", "team-a", "acme", "main", "tenant-us-1") + RecordBranchTarget("team-a", "from-eu", "team-a", "acme-main", "tenant-eu-1") + RecordBranchTarget("team-a", "from-us", "team-a", "acme-main", "tenant-us-1") for _, cluster := range []string{"tenant-eu-1", "tenant-us-1"} { value, ok := CollectInt64Sum(reader, branchTargetsMetric, map[string]string{ - "provider_namespace": "team-a", - "provider_name": "acme", - "branch": "main", - "source_cluster": cluster, + "gitbranch_namespace": "team-a", + "gitbranch_name": "acme-main", + "source_cluster": cluster, }) require.True(t, ok, "cluster %s must be reachable from the branch's own labels", cluster) assert.Equal(t, int64(1), value) @@ -164,8 +162,8 @@ func TestBranchTargets_RerecordReplacesTheClusterRatherThanAdding(t *testing.T) require.NoError(t, err) defer ForgetBranchTarget("team-a", "mirror") - RecordBranchTarget("team-a", "mirror", "team-a", "acme", "main", "tenant-eu-1") - RecordBranchTarget("team-a", "mirror", "team-a", "acme", "main", "tenant-eu-1") + RecordBranchTarget("team-a", "mirror", "team-a", "acme-main", "tenant-eu-1") + RecordBranchTarget("team-a", "mirror", "team-a", "acme-main", "tenant-eu-1") value, ok := branchTargetSeries(t, reader, "team-a", "mirror") require.True(t, ok) @@ -173,7 +171,7 @@ func TestBranchTargets_RerecordReplacesTheClusterRatherThanAdding(t *testing.T) } // Two GitTargets with the same name in different namespaces are different objects writing to -// different providers. Keying on name alone would collapse them, and the mapping would name the +// different GitBranches. Keying on name alone would collapse them, and the mapping would name the // wrong tenant's target for a branch failure. func TestBranchTargets_NamespaceIsPartOfTheIdentity(t *testing.T) { reader, err := InitTestExporter() @@ -181,13 +179,13 @@ func TestBranchTargets_NamespaceIsPartOfTheIdentity(t *testing.T) { defer ForgetBranchTarget("team-a", "mirror") defer ForgetBranchTarget("team-b", "mirror") - RecordBranchTarget("team-a", "mirror", "team-a", "acme", "main", "default") - RecordBranchTarget("team-b", "mirror", "team-b", "globex", "main", "default") + RecordBranchTarget("team-a", "mirror", "team-a", "acme-main", "default") + RecordBranchTarget("team-b", "mirror", "team-b", "globex-main", "default") value, ok := CollectInt64Sum(reader, branchTargetsMetric, map[string]string{ "gittarget_namespace": "team-b", "gittarget_name": "mirror", - "provider_name": "globex", + "gitbranch_name": "globex-main", }) require.True(t, ok) assert.Equal(t, int64(1), value) diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index 1e4963938..df5230ea1 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -82,9 +82,11 @@ var ( // change rather than an instrument. Read rates and ratios from it, not exact request counts. CommitRequestsTotal metric.Int64Counter // GitCommitsTotal counts commit batches that REACHED THE REMOTE, labelled by the recording - // BranchWorker's {provider_namespace, provider_name, branch, author_kind} identity plus + // BranchWorker's identity {gitbranch_namespace, gitbranch_name, provider_namespace, + // provider_name} (gitbranch_* and provider_* below) plus author_kind and // message_source. Live, snapshot and resync paths all feed this one counter; message_source - // (commit_request / live / reconcile) is what tells them apart. It counts commits that USED a + // (live / reconcile / commit_request / commit_request_framed / commit_request_fallback) is what + // tells them apart. It counts commits that USED a // request-supplied message, not CommitRequests: a request omitting spec.message renders through // liveTemplate and counts as live. // @@ -98,7 +100,7 @@ var ( // PlacementsTotal counts new-file placements resolved for a resource with no document in // Git yet — the only case placement runs for — labelled by {source, disposition, // gittarget_namespace, gittarget_name, group, version, resource}. source is which - // mechanism chose the path (by_type / default / kustomize_root / canonical) and disposition is what + // mechanism chose the path (by_type / default / kustomize_root / builtin) and disposition is what // it did with it (new_file / appended). // // It exists because sibling inference was deleted: a @@ -177,7 +179,7 @@ var ( // GitDocumentsTotal. WatchEventsTotal metric.Int64Counter // WatchEventHandlingSeconds records how long a target watch stream was BUSY on one event, - // labelled by {group, version, resource}: the whole of routeLiveTargetWatchEvent, attribution + // labelled by {source_cluster, group, version, resource}: the whole of routeLiveTargetWatchEvent, attribution // wait included. // // This is the head-of-line signal, and it is a proven failure rather than a theoretical one: a @@ -194,14 +196,15 @@ var ( // rate(_sum[5m]), the fraction of wall time the stream spent unavailable; approaching 1 means // events are queueing behind it. WatchEventHandlingSeconds metric.Float64Histogram - // WatchSessionsEndedTotal counts watch sessions that ended, labelled by {group, version, - // resource, reason}: `expired` (the resourceVersion fell out of history — 410 pressure, which - // forces a full replay), `error`, or `stopped` (the plan retired the stream, which is routine). + // WatchSessionsEndedTotal counts watch sessions that ended, labelled by {source_cluster, group, + // version, resource, reason}: `closed` (the API server's routine watch timeout, reconnected from the + // cursor), `expired` (the resourceVersion fell out of history — 410 pressure, which forces a + // full replay), `error`, or `stopped` (the plan retired the stream, which is routine). // A restart storm is a rebuild storm, and a rebuild walks the whole type. WatchSessionsEndedTotal metric.Int64Counter // WatchReplayDurationSeconds records how long an initial-events replay took to reach - // initial-events-end, labelled by {group, version, resource}. It is what a 410 storm actually - // charges: the cost of every rebuild it forces. + // initial-events-end, labelled by {source_cluster, group, version, resource}. It is what a 410 + // storm actually charges: the cost of every rebuild it forces. WatchReplayDurationSeconds metric.Float64Histogram // WatchRecoveryTotal counts completed watch recoveries, labelled by {gittarget_namespace, @@ -228,11 +231,11 @@ var ( // restart-reconcile e2e spec; treat the name and labels as a public observability contract. WatchRecoveryTotal metric.Int64Counter - // GitCommitFailuresTotal counts work that was routed to a worker and then died before it - // could ever be pushed, labelled by {provider_namespace, provider_name, branch, kind, reason}. - // kind is `window` (an ordinary live commit window) or `atomic` (a snapshot/resync request); - // reason is `refused` (the acceptance gate or a write-boundary precondition rejected the plan, - // which needs a human to fix the Git path) or `error` (a write fault). + // GitCommitFailuresTotal counts live commit windows that were routed to a worker and then died + // before they could ever be pushed, labelled by {gitbranch_*, provider_*, reason}. A resync's + // failure is git_resync_failures_total instead. reason is `refused` (the acceptance gate or a + // write-boundary precondition rejected the plan, which needs a human to fix the Git path) or + // `error` (a write fault). // // This was the largest remaining hole. A commit failure drops the whole window — the events are // already lost to the failed flush — and it happens AFTER routing and BEFORE pushing, so @@ -240,18 +243,17 @@ var ( // every object in that window until the next resync re-derives them, and until now the only // trace was a log line (or, for a refusal, a GitTarget condition nobody is alerting on). GitCommitFailuresTotal metric.Int64Counter - // GitWritesDroppedTotal counts the live events and atomic writes a branch worker dropped - // because their GitTarget's render fidelity was not established, labelled by - // {provider_namespace, provider_name, branch, kind, reason}. kind is `event` (one live event) - // or `atomic` (an atomic write request); reason is `render_fidelity_pending` (a scope of the - // GitTarget has not reported under its current revision yet) or `render_fidelity_diverged` (a - // rendered value does not match the live object). A dropped write is not retried: the edit is - // missing from Git until the object changes again or its collection replays. + // GitWritesDroppedTotal counts the live events a branch worker dropped because their GitTarget's + // render fidelity was not established, labelled by {gitbranch_*, provider_*, reason}. reason is + // `render_fidelity_pending` (a scope of the GitTarget has not reported under its current + // revision yet) or `render_fidelity_diverged` (a rendered value does not match the live + // object). A dropped write is not retried: the edit is missing from Git until the object + // changes again or its collection replays. GitWritesDroppedTotal metric.Int64Counter // GitCommitWindowsTotal counts closed commit windows, labelled by {gittarget_namespace, // gittarget_name, close_reason, timer_source}. close_reason says what ended the window // (idle_timeout, max_duration, attach_next, identity_change, buffer_limit, resync_before_apply, - // atomic_before_apply, shutdown, foreign_commit for a Git host's push notice, apply_due for the + // shutdown, foreign_commit for a Git host's push notice, apply_due for the // flush before an apply); timer_source says whose timers were in effect: the GitTarget's, // or a CommitRequest's that attached and replaced them. Every window is counted exactly once, at // the start of its finalize, so a window that then produces no diff or fails to commit is counted @@ -261,27 +263,28 @@ var ( // to log lines. timer_source keeps explicit saves from distorting the tuning of a target's own // defaults. GitCommitWindowsTotal metric.Int64Counter - // GitPushesTotal counts push CYCLES at their terminal end, labelled by {provider_namespace, - // provider_name, branch, outcome} where outcome is `pushed` or `failed`. A cycle that exhausts + // GitPushesTotal counts push CYCLES at their terminal end, labelled by {gitbranch_namespace, + // gitbranch_name, provider_*, outcome} where outcome is `pushed` or `failed`. A cycle that exhausts // its replay retries was previously a log line and nothing else: the mirror stops advancing and // every other metric reads healthy, because commits are still being created locally. GitPushesTotal metric.Int64Counter // GitPushRetriesTotal counts replay rounds inside a push cycle, labelled by - // {provider_namespace, provider_name, branch, reason}. reason has one value, `remote_moved`: + // {gitbranch_*, provider_*, reason}. reason has one value, `remote_moved`: // the remote branch moved, so the pending writes are rebuilt on the new head and re-pushed. // Every other rejection ends the cycle rather than retrying it, and is a `failed` outcome on // GitPushesTotal. A retry is not a terminal outcome, which is why it is its own counter rather // than a third value there; rate(retries)/rate(pushes) is the contention signal. GitPushRetriesTotal metric.Int64Counter // GitFetchesTotal counts every call that runs a SmartFetch against the remote, labelled by - // {provider_namespace, provider_name, branch, reason}. reason is `bootstrap` (the worker's - // first contact with the repository), `publication` (the fetch at the head of a publication - // cycle), `recovery` (a cycle that had to re-establish its base), `contention` (the reset onto the + // {gitbranch_*, provider_*, reason}. reason is + // `publication` (the fetch at the head of a publication cycle, which is also a worker's first + // contact with the repository), `recovery` (a cycle that had to re-establish its base), `contention` (the reset onto the // new tip after a rejection, which is one per confirmed rejection because the rejection itself // carries the remote's hash), `push_failure_probe` (the // lookup after a push that failed before the remote said anything, which is credentials or - // connectivity rather than another writer) or `forced_recheck` (an operator or controller - // asking the worker to re-read Git, and the snapshot a resync judges against). + // connectivity rather than another writer), `forced_recheck` (an operator or controller + // asking the worker to re-read Git, and the snapshot a resync judges against) or `refresh` (the + // periodic top-up of an idle branch, or a push-receiver delivery, when the branch had moved). // // Nothing else in the metric surface can tell you whether a fetch happened. GitPushesTotal // counts cycles and GitPushRetriesTotal counts replays, and both read identically whether the @@ -294,7 +297,7 @@ var ( // exactly on the contention path an operator is looking hardest at. GitFetchesTotal metric.Int64Counter // GitPushDurationSeconds records one push cycle's wall time, labelled by - // {provider_namespace, provider_name, branch}. Retries are inside the measurement on purpose: + // {gitbranch_*, provider_*}. Retries are inside the measurement on purpose: // what an operator wants is how long it took the mirror to accept the work, not how fast one // attempt was. GitPushDurationSeconds metric.Float64Histogram @@ -307,7 +310,7 @@ var ( // set costs bucket count + 2 series. GitCommitWindowDurationSeconds metric.Float64Histogram // GitQueueRefusalsTotal counts work the branch worker refused at enqueue, labelled by - // {provider_namespace, provider_name, branch, kind} where kind is `write`, `attach`, `resync` + // {gitbranch_*, provider_*, kind} where kind is `write`, `attach`, `resync` // or `refresh`. Its queue was full, its intake was paused through an outage, or it was // stopping. A refused item is not lost: its producer keeps it and offers it again (the watch // keeps its cursor and delivers the event again, the controller re-sends a save, a resync is @@ -321,7 +324,7 @@ var ( GitQueueRefusalsTotal metric.Int64Counter // GitMaterializationFailuresTotal counts attempts to commit decided writes that stopped because // the remote could not be reached, or the parent branch a new write branch is created from is - // missing, labelled by {provider_namespace, provider_name, branch, reason} where reason is + // missing, labelled by {gitbranch_*, provider_*, reason} where reason is // `unreachable` or `parent_unavailable`. Nothing is lost: the writes stay pending for the // retry. It is the failure before a push cycle starts, which git_pushes_total cannot see. GitMaterializationFailuresTotal metric.Int64Counter @@ -332,15 +335,23 @@ var ( // Manager.recordWatchRecovery), so a failed background apply is otherwise // only logged. This counter makes those failures observable/alertable without // triggering an immediate re-gather. Labelled by {gittarget_namespace, - // gittarget_name}; a sustained increase means snapshots are not committing and the - // folder is relying on steady-state events to catch up. + // gittarget_name, reason}. reason is `error` (the commit failed) or `timeout` (no answer in + // time), the two that mean the snapshot did not land; `refused` (the worker refused it at + // intake, also git_queue_refusals_total{kind="resync"}, and the stream gathers it again) or + // `deferred` (the remote could not be reached; it stays pending, also + // git_materialization_failures_total). A resync's commit failure is not + // git_commit_failures_total, which counts live windows only. A sustained + // error or timeout rate means snapshots are not committing and the folder is relying on + // steady-state events to catch up. GitResyncFailuresTotal metric.Int64Counter // AuditEventsTotal is the single per-event census: every successfully decoded, converted, and - // validated audit event increments it exactly once, labelled by {outcome, category, group, - // version, resource, verb}. Audit is attribution-only — it names the author of a watch-observed - // change; it never carries object state. Liveness = sum(...) > 0; the e2e invariant gates on - // category="error" == 0. + // validated audit event increments it exactly once, labelled by {audit_route, outcome, category, + // group, version, resource, verb}. audit_route is the route the event was posted under when a + // KubeProvider declares it, "unrecognized" when none does, and "none" for an event on the shared + // endpoint that names no route: a producer-chosen value is never a label unchecked. Audit is + // attribution-only — it names the author of a watch-observed change; it never carries object + // state. Liveness = sum(...) > 0; the e2e invariant gates on category="error" == 0. AuditEventsTotal metric.Int64Counter // AuditEventListDurationSeconds times every request at /audit-webhook, labelled by bounded // outcome: bad_method, bad_path and bare_endpoint_disabled for requests refused at the door, @@ -362,8 +373,9 @@ var ( // Together they say how much of what is published is ever used. They are NOT subtractable: // written counts every type, matched only the streams this process follows. AttributionFactsTotal metric.Int64Counter - // AttributionResolutionWaitSeconds records resolver wait time by {tier, event_kind, group, - // version, resource}. event_kind is write or removal, and the split is load-bearing: a removal + // AttributionResolutionWaitSeconds records resolver wait time by {tier, event_kind}; the type is + // on AttributionResolutionsTotal only, because a histogram pays its bucket count per label + // combination. event_kind is write or removal, and the split is load-bearing: a removal // holds a fallback and keeps waiting where a write does not, so the removal wait is the number // --author-attribution-grace is tuned from. AttributionResolutionWaitSeconds metric.Float64Histogram @@ -445,7 +457,7 @@ var ( // choosing the per-target deadline. WatchPlanPassDurationSeconds metric.Float64Histogram // WatchPlanTriggersTotal counts triggers by {reason, coalesced}. reason is declare, - // rule_change, shared_refresh or periodic, and shows which source is noisy. coalesced is + // rule_change, shared_refresh, periodic or worker_replaced, and shows which source is noisy. coalesced is // "true" when the trigger landed on a GitTarget that was ALREADY dirty, which is the proof // that the silence window does what it claims: one `kubectl apply` of a GitTarget and four // WatchRules should show four coalesced triggers and one pass. @@ -454,7 +466,8 @@ var ( // metric's business instead of a division across two. WatchPlanTriggersTotal metric.Int64Counter - // SecretEncryptionsTotal counts Secret encryption decisions, labelled by bounded outcome: + // SecretEncryptionsTotal counts Secret encryption decisions, labelled by the GitTarget the write + // was for {gittarget_namespace, gittarget_name} and a bounded outcome: // "encrypted" (the encryptor ran and produced ciphertext), "failed" (it ran and errored, and // the write is rejected), or "cached" (already-encrypted content was reused because the // sensitive marker was unchanged). @@ -485,12 +498,11 @@ var ( // otel_metric_overflow="true". const cardinalityLimit = 15000 -// InitOTLPExporter initializes the OTLP-to-Prometheus bridge. -func InitOTLPExporter(_ context.Context) (func(context.Context) error, error) { - fmt.Println("Initializing OTLP exporter") - - // Create a Prometheus exporter that bridges OTLP metrics to Prometheus - // Configure it to use the controller-runtime registry. +// InitPrometheusExporter registers every instrument on an OpenTelemetry meter provider whose only +// reader is the SDK's Prometheus exporter on controller-runtime's registry, so the instruments are +// served from the metrics server's /metrics beside controller-runtime's own. Nothing is pushed: the +// returned shutdown has nothing to flush and returns nil, however often it is called. +func InitPrometheusExporter(_ context.Context) (func(context.Context) error, error) { exporter, err := prometheus.New( prometheus.WithRegisterer(metrics.Registry), ) @@ -512,10 +524,7 @@ func InitOTLPExporter(_ context.Context) (func(context.Context) error, error) { return nil, err } - return func(_ context.Context) error { - fmt.Println("Shutting down OTLP exporter") - return nil - }, nil + return func(context.Context) error { return nil }, nil } // InitTestExporter wires the global instruments to a meter provider backed by a @@ -608,10 +617,10 @@ func registerCounters() error { {"gitopsreverser_watch_plan_triggers_total", &WatchPlanTriggersTotal}, {"gitopsreverser_apply_requests_total", &ApplyRequestsTotal}, {"gitopsreverser_apply_revision_mismatches_total", &ApplyRevisionMismatchesTotal}, - {"gitopsreverser_claimed_resource_reconciled_total", &ClaimedResourceReconciledTotal}, + {"gitopsreverser_apply_claimed_resource_reconciled_total", &ClaimedResourceReconciledTotal}, {"gitopsreverser_apply_window_collisions_total", &ApplyWindowCollisionsTotal}, {"gitopsreverser_apply_echoes_total", &ApplyEchoesTotal}, - {"gitopsreverser_stale_snapshots_total", &StaleSnapshotsTotal}, + {"gitopsreverser_apply_stale_snapshots_total", &StaleSnapshotsTotal}, {"gitopsreverser_apply_ownership_conflicts_total", &ApplyOwnershipConflictsTotal}, {"gitopsreverser_receiver_deliveries_total", &ReceiverDeliveriesTotal}, } @@ -746,7 +755,7 @@ func registerObservableGauges() error { {"gitopsreverser_resource_condition", GaugeResourceCondition}, {"gitopsreverser_git_branch_targets", GaugeGitBranchTargets}, {"gitopsreverser_apply_operations_unresolved", GaugeApplyOperationsUnresolved}, - {"gitopsreverser_appliers_held", GaugeAppliersHeld}, + {"gitopsreverser_apply_appliers_held", GaugeAppliersHeld}, } for _, o := range observable { if _, err := otelMeter.Int64ObservableGauge( @@ -764,6 +773,6 @@ func registerObservableGauges() error { SetGaugeSource(GaugeResourceCondition, resourceConditionSamples) // git_branch_targets is the same case: package state, always safe to read, empty until a // GitTarget reconcile publishes into it. - SetGaugeSource(GaugeGitBranchTargets, branchTargetSamples) + SetGaugeSource(GaugeGitBranchTargets, branchTargets.samples) return nil } diff --git a/internal/telemetry/exporter_test.go b/internal/telemetry/exporter_test.go index 16eef4a13..e8eaf6c00 100644 --- a/internal/telemetry/exporter_test.go +++ b/internal/telemetry/exporter_test.go @@ -20,10 +20,10 @@ func metricAttrs(key, value string) metric.MeasurementOption { return metric.WithAttributes(attribute.String(key, value)) } -func TestInitOTLPExporter_Success(t *testing.T) { +func TestInitPrometheusExporter_Success(t *testing.T) { ctx := context.Background() - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) assert.NotNil(t, shutdownFunc) @@ -60,7 +60,7 @@ func TestInitOTLPExporter_Success(t *testing.T) { func TestMetricsInitialization(t *testing.T) { ctx := context.Background() - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) defer func() { if shutdownFunc != nil { @@ -110,7 +110,7 @@ func TestMeterInitialization(t *testing.T) { func TestAuditPipelineMetricUsage(t *testing.T) { ctx := context.Background() - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) defer func() { if shutdownFunc != nil { @@ -129,7 +129,7 @@ func TestAuditPipelineMetricUsage(t *testing.T) { func TestAPICatalogMetricUsage(t *testing.T) { ctx := context.Background() - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) defer func() { if shutdownFunc != nil { @@ -151,7 +151,7 @@ func TestAPICatalogMetricUsage(t *testing.T) { func TestConcurrentMetricsUsage(t *testing.T) { ctx := context.Background() - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) defer func() { if shutdownFunc != nil { @@ -191,7 +191,7 @@ func TestConcurrentMetricsUsage(t *testing.T) { func TestHistogramMetricBehavior(t *testing.T) { ctx := context.Background() - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) defer func() { if shutdownFunc != nil { @@ -229,7 +229,7 @@ func TestMetricsErrorHandling(t *testing.T) { func TestShutdownFunction(t *testing.T) { ctx := context.Background() - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) require.NotNil(t, shutdownFunc) @@ -244,7 +244,7 @@ func TestShutdownFunction(t *testing.T) { func TestMetricsAfterShutdown(t *testing.T) { ctx := context.Background() - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) assert.NotPanics(t, func() { @@ -337,7 +337,7 @@ func TestNoOpMeterProvider(t *testing.T) { otel.SetMeterProvider(noopProvider) defer otel.SetMeterProvider(nil) - shutdownFunc, err := InitOTLPExporter(ctx) + shutdownFunc, err := InitPrometheusExporter(ctx) require.NoError(t, err) assert.NotNil(t, shutdownFunc) diff --git a/internal/telemetry/gauges.go b/internal/telemetry/gauges.go index c2feb0d72..72359323c 100644 --- a/internal/telemetry/gauges.go +++ b/internal/telemetry/gauges.go @@ -60,14 +60,14 @@ const ( // target's join series ends at the next scrape rather than attributing a live branch's // failures to an object that is gone. See branch_targets.go. GaugeGitBranchTargets = "git_branch_targets" - // GaugeApplyOperationsUnresolved is how many apply operations on a branch did not complete + // GaugeApplyOperationsUnresolved is how many apply operations on a GitBranch did not complete // within the dispatcher's budget and are still followed, each holding its applier's apply // window open and its next attempt back. Read from what the branch worker's loop last published. GaugeApplyOperationsUnresolved = "apply_operations_unresolved" - // GaugeAppliersHeld is how many appliers on a branch have a reason pending that the initial + // GaugeAppliersHeld is how many appliers on a GitBranch have a reason pending that the initial // snapshot barrier holds back: a GitTarget they cover is not mirrored yet, or its folder is not on // the branch. Read from what the branch worker's loop last published. - GaugeAppliersHeld = "appliers_held" + GaugeAppliersHeld = "apply_appliers_held" ) var ( diff --git a/internal/telemetry/gauges_test.go b/internal/telemetry/gauges_test.go index 7ed2ce4a3..769d9f185 100644 --- a/internal/telemetry/gauges_test.go +++ b/internal/telemetry/gauges_test.go @@ -19,18 +19,18 @@ func TestObservableGauge_ReadsTheSourceAtCollectionTime(t *testing.T) { depth := int64(0) SetGaugeSource(GaugeGitQueueDepth, func() []GaugeSample { - return []GaugeSample{{Value: depth, Attrs: []attribute.KeyValue{attribute.String("branch", "main")}}} + return []GaugeSample{{Value: depth, Attrs: []attribute.KeyValue{attribute.String("gitbranch_name", "main")}}} }) defer SetGaugeSource(GaugeGitQueueDepth, nil) - value, ok := CollectInt64Sum(reader, "gitopsreverser_git_queue_depth", map[string]string{"branch": "main"}) + value, ok := CollectInt64Sum(reader, "gitopsreverser_git_queue_depth", map[string]string{"gitbranch_name": "main"}) require.True(t, ok) require.Equal(t, int64(0), value) // Nothing republishes. The next scrape must still see the new value. depth = 7 - value, ok = CollectInt64Sum(reader, "gitopsreverser_git_queue_depth", map[string]string{"branch": "main"}) + value, ok = CollectInt64Sum(reader, "gitopsreverser_git_queue_depth", map[string]string{"gitbranch_name": "main"}) require.True(t, ok) assert.Equal(t, int64(7), value) } diff --git a/internal/telemetry/object_series.go b/internal/telemetry/object_series.go new file mode 100644 index 000000000..dd6c8a97c --- /dev/null +++ b/internal/telemetry/object_series.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +package telemetry + +import ( + "sync" + + "go.opentelemetry.io/otel/attribute" +) + +// objectSeries holds always-1 gauge series keyed by one configuration object, read at scrape time: +// the shape of git_branch_targets. Each object publishes one series, its own identity followed by +// what is recorded for it. +// +// Observable rather than pushed for the reason resource_condition is: these families need series +// that STOP. A pushed gauge latches its last value, so a deleted object would stay published for +// the life of the process. Reading a map at scrape time makes the delete a map delete. +type objectSeries struct { + // namespaceKey and nameKey are the label keys the object's identity is published under. Always + // prefixed, never a bare namespace/name: a Prometheus pod scrape with honor_labels=false renames + // those to exported_* and puts the scraped pod's own in their place. + namespaceKey string + nameKey string + + mu sync.RWMutex + entries map[objectKey][]attribute.KeyValue +} + +// objectKey identifies the object one published series is about. +type objectKey struct { + namespace string + name string +} + +func newObjectSeries(namespaceKey, nameKey string) *objectSeries { + return &objectSeries{ + namespaceKey: namespaceKey, + nameKey: nameKey, + entries: map[objectKey][]attribute.KeyValue{}, + } +} + +// record publishes one object's labels, replacing any previous entry for it, so a re-record can +// never leave two series for one object. +func (s *objectSeries) record(namespace, name string, attrs ...attribute.KeyValue) { + s.mu.Lock() + defer s.mu.Unlock() + s.entries[objectKey{namespace: namespace, name: name}] = attrs +} + +// forget stops publishing one object's series. +func (s *objectSeries) forget(namespace, name string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.entries, objectKey{namespace: namespace, name: name}) +} + +// samples renders one sample per object. It takes only its own short-lived lock, which no +// reconcile holds across I/O. +func (s *objectSeries) samples() []GaugeSample { + s.mu.RLock() + defer s.mu.RUnlock() + + samples := make([]GaugeSample, 0, len(s.entries)) + for key, attrs := range s.entries { + samples = append(samples, GaugeSample{ + Value: 1, + Attrs: append([]attribute.KeyValue{ + attribute.String(s.namespaceKey, key.namespace), + attribute.String(s.nameKey, key.name), + }, attrs...), + }) + } + return samples +} diff --git a/internal/telemetry/resource_condition.go b/internal/telemetry/resource_condition.go index 4e755534a..90d0c7d82 100644 --- a/internal/telemetry/resource_condition.go +++ b/internal/telemetry/resource_condition.go @@ -32,10 +32,10 @@ import ( // Publishing from a map read at scrape time makes the delete a map delete: ForgetResourceConditions // drops the key and the series stops. There is no gauge value to "unset". // -// It is also the only family here keyed on object identity, which -// docs/design/metrics-observability-plan.md §7.1 permits for exactly this population and nothing -// else: these are CONFIG objects, bounded by how many a human wrote, not the watched objects the -// operator mirrors, which are data and unbounded. The label keys are prefixed — resource_namespace, +// It is keyed on object identity, which docs/design/metrics-observability-plan.md §7.1 permits for +// this population as it does for the gittarget_*, gitbranch_* and claim_* labels elsewhere: these +// are CONFIG objects, bounded by how many a human wrote, not the watched objects the operator +// mirrors, which are data and unbounded. The label keys are prefixed — resource_namespace, // not a bare namespace — because a Prometheus pod scrape with honor_labels=false overwrites a bare // `namespace` attribute with the scraped pod's own. diff --git a/internal/watch/author_resolver.go b/internal/watch/author_resolver.go index 08897447e..3f47d20a8 100644 --- a/internal/watch/author_resolver.go +++ b/internal/watch/author_resolver.go @@ -213,8 +213,9 @@ const ( // // The two instruments carry DIFFERENT label sets on purpose. actor_kind is a property of the answer // and belongs on the census; event_kind is a property of the question and only changes what the -// wait means. Putting both on both would multiply a histogram that already carries the type triple -// by six for no reading anyone would make. +// wait means. The type triple is on the census only: a histogram pays its bucket count for every +// label combination, which made this the largest family in the cardinality budget, and the wait a +// type's attribution costs is read from the census's tier="absent" rate for that type. func recordAttributionResolution( ctx context.Context, query AuthorQuery, @@ -235,7 +236,9 @@ func recordAttributionResolution( } if telemetry.AttributionResolutionWaitSeconds != nil { telemetry.AttributionResolutionWaitSeconds.Record(ctx, wait.Seconds(), metric.WithAttributes( - append(typeAttrs, attribute.String("event_kind", attributionEventKind(query)))...)) + attribute.String("tier", string(tier)), + attribute.String("event_kind", attributionEventKind(query)), + )) } } diff --git a/internal/watch/author_resolver_test.go b/internal/watch/author_resolver_test.go index d7944abde..9064d9560 100644 --- a/internal/watch/author_resolver_test.go +++ b/internal/watch/author_resolver_test.go @@ -359,3 +359,21 @@ func TestAuthorResolver_AuthorlessMatchProvesTheRouteIsAlive(t *testing.T) { assert.False(t, warn, "a route that has delivered a fact must never be accused of delivering none") } } + +// The wait histogram carries no type: a histogram pays its bucket count for every label +// combination, and the per-type question is answered by the census beside it. +func TestAuthorResolver_WaitHistogramCarriesNoType(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + lookup := &fakeLookup{resolution: queue.AuthorResolution{Result: queue.AttributionAbsent}} + r := NewAuthorResolver(lookup, 0, logr.Discard(), nil) + _, _ = r.ResolveAuthor(context.Background(), resolverQuery("prod-eu-1", "uid-1", "999", false)) + + _, typed := telemetry.CollectHistogramCount(reader, "gitopsreverser_attribution_resolution_wait_seconds", + map[string]string{"resource": resolverGVR.Resource}) + assert.False(t, typed, "the wait histogram must not carry the type triple") + _, counted := telemetry.CollectInt64Sum(reader, "gitopsreverser_attribution_resolutions_total", + map[string]string{"tier": string(queue.AttributionAbsent), "resource": resolverGVR.Resource}) + assert.True(t, counted, "the census still counts the resolution, by type") +} diff --git a/internal/watch/event_router.go b/internal/watch/event_router.go index 062d491fc..dd26e83e8 100644 --- a/internal/watch/event_router.go +++ b/internal/watch/event_router.go @@ -212,16 +212,40 @@ func (r *EventRouter) commitRequestWorker( return nil, gitBranch.Spec.Branch, nil } -// recordBackgroundResyncFailure counts a fire-and-forget resync whose apply failed or -// timed out at the worker, so the failure is observable even though delivery was already -// marked on enqueue. No-op until the counter is registered. -func (r *EventRouter) recordBackgroundResyncFailure(gitDest types.ResourceReference) { +// git_resync_failures_total reasons. Only error and timeout are a snapshot that did not land: a +// refused one is gathered again by its stream, and a deferred one stays pending at the worker. +const ( + resyncFailureRefused = "refused" + resyncFailureDeferred = "deferred" + resyncFailureTimeout = "timeout" + resyncFailureError = "error" +) + +// resyncFailureReason classifies a resync's error for git_resync_failures_total. A refusal at +// intake is also git_queue_refusals_total{kind="resync"}, and a deferral also +// git_materialization_failures_total: the reason keeps those two views apart from a real failure. +func resyncFailureReason(err error) string { + switch { + case errors.Is(err, git.ErrFinalizeQueueFull): + return resyncFailureRefused + case errors.Is(err, git.ErrResyncDeferred): + return resyncFailureDeferred + default: + return resyncFailureError + } +} + +// recordBackgroundResyncFailure counts a fire-and-forget resync whose apply failed, was refused or +// deferred, or timed out at the worker, so the failure is observable even though delivery was +// already marked on enqueue. No-op until the counter is registered. +func (r *EventRouter) recordBackgroundResyncFailure(gitDest types.ResourceReference, reason string) { if telemetry.GitResyncFailuresTotal == nil { return } telemetry.GitResyncFailuresTotal.Add(context.Background(), 1, metric.WithAttributes( attribute.String("gittarget_namespace", gitDest.Namespace), attribute.String("gittarget_name", gitDest.Name), + attribute.String("reason", reason), )) } @@ -416,7 +440,7 @@ func (r *EventRouter) drainScopedResync( r.WatchManager.markSnapshotSettled(gitDest, collection, snapshots) } r.Log.Error(nil, "per-type "+kind+" timed out", "gitDest", gitDest.String(), "collection", collection.String()) - r.recordBackgroundResyncFailure(gitDest) + r.recordBackgroundResyncFailure(gitDest, resyncFailureTimeout) } } @@ -481,7 +505,7 @@ func (r *EventRouter) handleScopedResyncError( return } r.Log.Error(err, "per-type "+kind+" failed", "gitDest", gitDest.String(), "collection", collection.String()) - r.recordBackgroundResyncFailure(gitDest) + r.recordBackgroundResyncFailure(gitDest, resyncFailureReason(err)) } func renderFidelityDivergence(refused *manifestanalyzer.AcceptanceRefusedError) manifestanalyzer.RenderDivergence { diff --git a/internal/watch/owner.go b/internal/watch/owner.go index 69c4e61ca..9768f7932 100644 --- a/internal/watch/owner.go +++ b/internal/watch/owner.go @@ -53,9 +53,8 @@ const ( sharedRefreshTimeout = 2 * time.Minute ) -// Trigger reasons, as they appear on the triggers-by-reason metric and in logs. There are three -// because there are three ways intent reaches the owner, and telling them apart is what shows -// which source is noisy. +// Trigger reasons, as they appear on the triggers-by-reason metric and in logs. There is one per +// way intent reaches the owner, and telling them apart is what shows which source is noisy. const ( // TriggerReasonDeclare is the GitTarget controller submitting a declaration. TriggerReasonDeclare = "declare" diff --git a/internal/watch/target_reconcile_metric_test.go b/internal/watch/target_reconcile_metric_test.go index a4a037008..837a4cc0f 100644 --- a/internal/watch/target_reconcile_metric_test.go +++ b/internal/watch/target_reconcile_metric_test.go @@ -3,12 +3,15 @@ package watch import ( + "errors" + "fmt" "testing" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/telemetry" "github.com/ConfigButler/gitops-reverser/internal/types" ) @@ -26,15 +29,27 @@ func TestRecordBackgroundResyncFailure_IncrementsPerGitTarget(t *testing.T) { r := &EventRouter{Log: logr.Discard()} gitDest := types.NewResourceReference("my-target", "my-ns") - r.recordBackgroundResyncFailure(gitDest) - r.recordBackgroundResyncFailure(gitDest) + r.recordBackgroundResyncFailure(gitDest, resyncFailureError) + r.recordBackgroundResyncFailure(gitDest, resyncFailureError) value, ok := telemetry.CollectInt64Sum(reader, resyncFailuresMetric, - map[string]string{"gittarget_namespace": "my-ns", "gittarget_name": "my-target"}) + map[string]string{"gittarget_namespace": "my-ns", "gittarget_name": "my-target", "reason": "error"}) require.True(t, ok, "expected a git_resync_failures_total sample") assert.Equal(t, int64(2), value) } +// The reason separates a snapshot that did not land from two that are counted elsewhere too: a +// resync refused at intake (git_queue_refusals_total) is gathered again, and a deferred one +// (git_materialization_failures_total) stays pending at the worker. +func TestResyncFailureReason_SeparatesRefusedAndDeferredFromFailed(t *testing.T) { + assert.Equal(t, resyncFailureRefused, resyncFailureReason(git.ErrFinalizeQueueFull)) + assert.Equal(t, resyncFailureRefused, + resyncFailureReason(fmt.Errorf("%w: the branch has paused intake", git.ErrFinalizeQueueFull))) + deferred := fmt.Errorf("%w: refresh remote before resync: %w", git.ErrResyncDeferred, errors.New("dial")) + assert.Equal(t, resyncFailureDeferred, resyncFailureReason(deferred)) + assert.Equal(t, resyncFailureError, resyncFailureReason(errors.New("write failed"))) +} + // recordWatchRecovery must increment the per-GitTarget counter and carry the gittarget_*, // group/resource and mode labels. // diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index fc5ee7551..dc6364bc7 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -574,7 +574,7 @@ func (m *Manager) runTargetWatch( stream.changes = newDesiredStateChangeFilter() for ctx.Err() == nil { err := m.targetWatchReplayAndStream(ctx, log, gitDest, stream, stream.hasReplayed()) - recordWatchSessionEnded(ctx, stream.key.GVR, sessionEndReason(ctx, err)) + recordWatchSessionEnded(ctx, m.clusterIDForGitTarget(gitDest), stream.key.GVR, sessionEndReason(ctx, err)) if ctx.Err() != nil { return } @@ -709,7 +709,7 @@ func (m *Manager) targetWatchReplayAndStream( // HERE because the wrapper swallows the sentinel and falls through to a fresh replay, so // the outer session-end recording never sees it — and `expired` is the reason that matters // most, since it is the one that forces a full rebuild of the type. - recordWatchSessionEnded(ctx, stream.key.GVR, sessionEndedExpired) + recordWatchSessionEnded(ctx, m.clusterIDForGitTarget(gitDest), stream.key.GVR, sessionEndedExpired) m.markTargetStreamState( gitDest, stream.key.Collection(), @@ -808,7 +808,7 @@ func (m *Manager) pumpTargetWatchSession( if replaying && !nextReplaying { // initial-events-end. This is what a 410 storm charges: the cost of every rebuild // it forces, which no other instrument can see. - recordWatchReplayDuration(ctx, stream.key.GVR, replayStarted) + recordWatchReplayDuration(ctx, m.clusterIDForGitTarget(gitDest), stream.key.GVR, replayStarted) m.recordWatchRecovery(gitDest, stream.key.GVR.Group, stream.key.GVR.Resource, recoveryModeReplay) } replaying = nextReplaying @@ -1158,7 +1158,7 @@ func (m *Manager) routeLiveTargetWatchEvent( // reported no occupancy at all. A stream is single-threaded, so time spent in this function is // time nothing else on that stream is being read, which is the whole signal. defer func(started time.Time) { - recordWatchEventHandling(ctx, stream.key.GVR, started) + recordWatchEventHandling(ctx, m.clusterIDForGitTarget(gitDest), stream.key.GVR, started) }(time.Now()) rv := targetWatchEventResourceVersion(ev) diff --git a/internal/watch/watch_event_metrics.go b/internal/watch/watch_event_metrics.go index 549fe6187..69be3655b 100644 --- a/internal/watch/watch_event_metrics.go +++ b/internal/watch/watch_event_metrics.go @@ -94,16 +94,27 @@ func recordWatchEvent( telemetry.WatchEventsTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) } +// streamAttrs is the {source_cluster, group, version, resource} identity of the per-type stream +// instruments. A type is served by one API server per cluster, and that server is what closes, +// expires and replays a watch, so the cluster is part of what a session is. It is the same +// source_cluster value watch_types and watch_streams_open carry. The GitTarget is deliberately not +// here: two of these are histograms, which pay bucket count + 2 series per label combination. +func streamAttrs(clusterID string, gvr schema.GroupVersionResource) []attribute.KeyValue { + return append([]attribute.KeyValue{attribute.String("source_cluster", clusterID)}, gvrAttrs(gvr)...) +} + // recordWatchEventHandling times how long the stream was busy on one event. // // Occupancy, not queue delay: see telemetry.WatchEventHandlingSeconds for why the wait itself // cannot be measured honestly from this side of a client-go watch channel. -func recordWatchEventHandling(ctx context.Context, gvr schema.GroupVersionResource, started time.Time) { +func recordWatchEventHandling( + ctx context.Context, clusterID string, gvr schema.GroupVersionResource, started time.Time, +) { if telemetry.WatchEventHandlingSeconds == nil { return } telemetry.WatchEventHandlingSeconds.Record(ctx, time.Since(started).Seconds(), - metric.WithAttributes(gvrAttrs(gvr)...)) + metric.WithAttributes(streamAttrs(clusterID, gvr)...)) } // sessionEndReason classifies why a watch session ended. @@ -136,20 +147,22 @@ func sessionEndReason(ctx context.Context, err error) string { } // recordWatchSessionEnded counts one watch session ending, under the reason it ended for. -func recordWatchSessionEnded(ctx context.Context, gvr schema.GroupVersionResource, reason string) { +func recordWatchSessionEnded(ctx context.Context, clusterID string, gvr schema.GroupVersionResource, reason string) { if telemetry.WatchSessionsEndedTotal == nil { return } - attrs := append(gvrAttrs(gvr), attribute.String("reason", reason)) + attrs := append(streamAttrs(clusterID, gvr), attribute.String("reason", reason)) telemetry.WatchSessionsEndedTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) } // recordWatchReplayDuration times one initial-events replay to initial-events-end. It is what a // 410 storm charges: the cost of every rebuild it forces. -func recordWatchReplayDuration(ctx context.Context, gvr schema.GroupVersionResource, started time.Time) { +func recordWatchReplayDuration( + ctx context.Context, clusterID string, gvr schema.GroupVersionResource, started time.Time, +) { if telemetry.WatchReplayDurationSeconds == nil { return } telemetry.WatchReplayDurationSeconds.Record(ctx, time.Since(started).Seconds(), - metric.WithAttributes(gvrAttrs(gvr)...)) + metric.WithAttributes(streamAttrs(clusterID, gvr)...)) } diff --git a/internal/watch/watch_event_metrics_test.go b/internal/watch/watch_event_metrics_test.go index 150d95d44..32351dda7 100644 --- a/internal/watch/watch_event_metrics_test.go +++ b/internal/watch/watch_event_metrics_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -116,3 +117,36 @@ func TestRecordWatchRecovery_CarriesNoVersionLabel(t *testing.T) { map[string]string{"version": "v1"}) assert.False(t, ok, "the recovery counter must not carry a version label") } + +// The per-type stream instruments name the source cluster, because the API server that closes, +// expires and replays a watch is that cluster's: two KubeProviders serving one type are two +// streams, and an expiry storm on one must not read as the other's. +func TestStreamInstruments_AreSplitBySourceCluster(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + gvr := schema.GroupVersionResource{Version: "v1", Resource: "configmaps"} + started := time.Now() + recordWatchSessionEnded(t.Context(), "team-a/eu", gvr, sessionEndedExpired) + recordWatchSessionEnded(t.Context(), "team-a/eu", gvr, sessionEndedExpired) + recordWatchSessionEnded(t.Context(), "team-a/us", gvr, sessionEndedClosed) + recordWatchReplayDuration(t.Context(), "team-a/eu", gvr, started) + recordWatchEventHandling(t.Context(), "team-a/us", gvr, started) + + expired, ok := telemetry.CollectInt64Sum(reader, "gitopsreverser_watch_sessions_ended_total", + map[string]string{"source_cluster": "team-a/eu", "resource": "configmaps", "reason": sessionEndedExpired}) + require.True(t, ok) + assert.Equal(t, int64(2), expired) + _, ok = telemetry.CollectInt64Sum(reader, "gitopsreverser_watch_sessions_ended_total", + map[string]string{"source_cluster": "team-a/us", "reason": sessionEndedExpired}) + assert.False(t, ok, "the other cluster's sessions did not expire") + + replays, ok := telemetry.CollectHistogramCount(reader, "gitopsreverser_watch_replay_duration_seconds", + map[string]string{"source_cluster": "team-a/eu", "resource": "configmaps"}) + require.True(t, ok) + assert.Equal(t, uint64(1), replays) + handled, ok := telemetry.CollectHistogramCount(reader, "gitopsreverser_watch_event_handling_seconds", + map[string]string{"source_cluster": "team-a/us", "resource": "configmaps"}) + require.True(t, ok) + assert.Equal(t, uint64(1), handled) +} diff --git a/internal/webhook/audit_handler.go b/internal/webhook/audit_handler.go index 3ebfde7a7..c3d05e8d8 100644 --- a/internal/webhook/audit_handler.go +++ b/internal/webhook/audit_handler.go @@ -198,6 +198,24 @@ type auditRoute struct { annotationKey string } +// routeLabel is the audit_route an event is counted under: its route when a KubeProvider declares +// it, so "whose audit stopped" is a selector. It reads the route the way resolveEventRoute does, +// without its side effects, because an event the accept gate drops is counted before it is routed. +func (h *AuditHandler) routeLabel(route auditRoute, event *auditv1.Event) string { + name := route.route + if route.annotationKey != "" && event != nil { + name = event.Annotations[route.annotationKey] + } + switch { + case name == "": + return outcome.RouteNone + case h.config.RouteHealth.RouteDeclared(name): + return name + default: + return outcome.RouteUnrecognized + } +} + // auditRouteForPath maps an accepted request path to the AUDIT ROUTE its facts belong to and // whether the path named one. /audit-webhook/ carries the route (named=true). The bare // /audit-webhook names none — it is the shared, annotation-routed endpoint, so it returns @@ -326,7 +344,7 @@ func (h *AuditHandler) processEvents(ctx context.Context, route auditRoute, even batches.add(result) } appended, err := h.publishFactBatches(ctx, batches) - h.recordAcceptedOutcomes(ctx, accepted, appended) + h.recordAcceptedOutcomes(ctx, route, accepted, appended) if err != nil { return err } @@ -344,11 +362,13 @@ func (h *AuditHandler) processEvents(ctx context.Context, route auditRoute, even // the only place it can be counted: no fact exists, so no fact-side counter can ever see it. func (h *AuditHandler) recordAcceptedOutcomes( ctx context.Context, + route auditRoute, accepted []acceptedFact, appended map[queue.FactStreamKey]struct{}, ) { for i := range accepted { - outcome.Record(ctx, accepted[i].event, acceptedOutcome(accepted[i], appended)) + outcome.Record(ctx, accepted[i].event, acceptedOutcome(accepted[i], appended), + h.routeLabel(route, accepted[i].event)) } } @@ -399,12 +419,12 @@ func (h *AuditHandler) processEvent( if !shouldForwardSubresource(&event) { // A non-/scale subresource (or an unmapped-verb subresource): dropped before recording. - outcome.Record(ctx, &event, outcome.NonScaleSubresource) + outcome.Record(ctx, &event, outcome.NonScaleSubresource, h.routeLabel(route, &event)) return acceptedFact{} } if decision := classifyAuditIngress(&event); !decision.Process { - outcome.Record(ctx, &event, outcome.Outcome(decision.Reason)) + outcome.Record(ctx, &event, outcome.Outcome(decision.Reason), h.routeLabel(route, &event)) log.V(1).Info("Dropped audit event before recording", "reason", decision.Reason, "gvr", extractGVR(&event), "auditID", event.AuditID) return acceptedFact{} @@ -541,7 +561,8 @@ func (h *AuditHandler) rejectUnroutableEvent( annotationKey string, sourceCluster string, ) { - outcome.Record(ctx, event, reason) + // Its only caller is an event that names no route. + outcome.Record(ctx, event, reason, outcome.RouteNone) log := logf.Log.WithName("audit-handler") fields := []any{ diff --git a/internal/webhook/audit_handler_test.go b/internal/webhook/audit_handler_test.go index 0b385d2f6..96420bd00 100644 --- a/internal/webhook/audit_handler_test.go +++ b/internal/webhook/audit_handler_test.go @@ -29,7 +29,7 @@ import ( // the steady-state code paths exercised. Per-test value assertions use a fresh // telemetry.InitTestExporter() reader. func TestMain(m *testing.M) { - if _, err := telemetry.InitOTLPExporter(context.Background()); err != nil { + if _, err := telemetry.InitPrometheusExporter(context.Background()); err != nil { panic("Failed to initialize metrics: " + err.Error()) } os.Exit(m.Run()) diff --git a/internal/webhook/audit_route_label_test.go b/internal/webhook/audit_route_label_test.go new file mode 100644 index 000000000..ac50d1e1c --- /dev/null +++ b/internal/webhook/audit_route_label_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package webhook + +import ( + "testing" + + "github.com/stretchr/testify/assert" + k8stypes "k8s.io/apimachinery/pkg/types" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + + "github.com/ConfigButler/gitops-reverser/internal/attribution" + "github.com/ConfigButler/gitops-reverser/internal/audit/outcome" +) + +// The audit census names a route only when a KubeProvider declares it: the producer picks the path +// segment or annotation, so an undeclared value collapses into one bucket instead of a series each. +func TestRouteLabel_IsBoundedByDeclaredRoutes(t *testing.T) { + health := &attribution.RouteHealth{} + provider := k8stypes.NamespacedName{Namespace: "team-a", Name: "prod"} + health.DeclareRoute(provider, "team-a.prod") + h := &AuditHandler{config: AuditHandlerConfig{RouteHealth: health, AuditRouteAnnotationKey: "route"}} + + assert.Equal(t, "team-a.prod", h.routeLabel(auditRoute{route: "team-a.prod"}, &auditv1.Event{})) + assert.Equal(t, outcome.RouteUnrecognized, h.routeLabel(auditRoute{route: "made-up"}, &auditv1.Event{})) + + shared := auditRoute{annotationKey: "route"} + annotated := &auditv1.Event{Annotations: map[string]string{"route": "team-a.prod"}} + assert.Equal(t, "team-a.prod", h.routeLabel(shared, annotated), "the shared endpoint reads it per event") + assert.Equal(t, outcome.RouteNone, h.routeLabel(shared, &auditv1.Event{}), "an event naming no route") + + health.ForgetRoute(provider) + assert.Equal(t, outcome.RouteUnrecognized, h.routeLabel(auditRoute{route: "team-a.prod"}, &auditv1.Event{}), + "a deleted provider's route is no longer a label value") +} diff --git a/test/e2e/commit_request_e2e_test.go b/test/e2e/commit_request_e2e_test.go index 34b34a2e0..706b67d57 100644 --- a/test/e2e/commit_request_e2e_test.go +++ b/test/e2e/commit_request_e2e_test.go @@ -186,11 +186,12 @@ var _ = Describe("Commit Request", Label("commit-request", "audit-consumer"), Or waitForMetric( fmt.Sprintf( `sum(gitopsreverser_git_branch_targets{gittarget_namespace=%q,gittarget_name=%q,`+ - `provider_namespace=%q,provider_name=%q,branch="main",source_cluster=%q}) `+ + `gitbranch_namespace=%q,gitbranch_name=%q,source_cluster=%q}) `+ `or vector(0)`, - testNs, gitTargetName, testNs, gitProvName, resolveE2ENamespace()+"/"+e2eKubeProviderName), + testNs, gitTargetName, testNs, gitBranchName(gitProvName, "main"), + resolveE2ENamespace()+"/"+e2eKubeProviderName), func(v float64) bool { return v == 1 }, - "the join series maps this GitTarget to the provider and branch its commits are labelled by", + "the join series maps this GitTarget to the GitBranch its commits are labelled by", ) By("cleaning up the test Deployment and CommitRequest") diff --git a/test/e2e/crd_lifecycle_e2e_test.go b/test/e2e/crd_lifecycle_e2e_test.go index e418bf923..ec538b832 100644 --- a/test/e2e/crd_lifecycle_e2e_test.go +++ b/test/e2e/crd_lifecycle_e2e_test.go @@ -277,7 +277,7 @@ var _ = Describe("CRD Lifecycle", Label("crd-lifecycle"), Serial, Ordered, func( // is the file-existence assertion below. gitopsreverser_git_commits_total counts // both commit paths (per-event and backfill resync), so this single metric // replaces the prior Or() of two operator-log substrings, and the - // provider_namespace label scopes it to this suite's GitProvider. + // gitbranch_namespace label scopes it to this suite's GitBranch. waitForCommitInNamespace(testNs) By("verifying CRD instance YAML file exists in Gitea repository") diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index dcd63a2be..064617f17 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -131,22 +131,22 @@ func ensurePrometheusClient() { } // waitForCommitInNamespace blocks until gitopsreverser_git_commits_total reports at least -// one commit for a BranchWorker whose GitProvider lives in providerNamespace. Because -// each e2e suite creates its GitProvider in a unique namespace (testNamespaceFor), this +// one commit for a BranchWorker whose GitBranch lives in namespace. Because each e2e +// suite creates its GitBranches in a unique namespace (testNamespaceFor), this // isolates the calling spec's commits from any other spec committing in parallel — the // robust replacement for scraping operator logs for "git commit created". The metric // helper's Eventually poll absorbs the ~5s Prometheus scrape interval, and the per-event // and backfill-resync commit paths both feed this one counter. -func waitForCommitInNamespace(providerNamespace string) { +func waitForCommitInNamespace(namespace string) { ensurePrometheusClient() query := fmt.Sprintf( - `sum(gitopsreverser_git_commits_total{provider_namespace=%q}) or vector(0)`, - providerNamespace, + `sum(gitopsreverser_git_commits_total{gitbranch_namespace=%q}) or vector(0)`, + namespace, ) waitForMetricWithTimeout( query, func(v float64) bool { return v > 0 }, - fmt.Sprintf("a commit was created for GitProvider namespace %q", providerNamespace), + fmt.Sprintf("a commit was created for a GitBranch in namespace %q", namespace), 45*time.Second, //nolint:mnd // matches the prior log-scrape Eventually budget ) } diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go index 307831ea6..9e24fc2d9 100644 --- a/test/e2e/new_branch_parent_e2e_test.go +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -229,8 +229,8 @@ data: } branchSeries := func(metric, extra string) string { - return fmt.Sprintf(`sum(%s{provider_namespace=%q,branch=%q%s}) or vector(0)`, - metric, testNs, writeBranch, extra) + return fmt.Sprintf(`sum(%s{gitbranch_namespace=%q,gitbranch_name=%q%s}) or vector(0)`, + metric, testNs, gitBranchName(providerName, writeBranch), extra) } pushes := func() float64 { GinkgoHelper() diff --git a/test/e2e/restart_reconcile_e2e_test.go b/test/e2e/restart_reconcile_e2e_test.go index eae27dc01..e5d01d6c3 100644 --- a/test/e2e/restart_reconcile_e2e_test.go +++ b/test/e2e/restart_reconcile_e2e_test.go @@ -224,8 +224,8 @@ var _ = Describe("Restart Reconcile Safety", Label("restart-reconcile"), Serial, waitForMetricWithTimeout( fmt.Sprintf( `sum(gitopsreverser_git_queue_depth`+ - `{provider_namespace=%q,provider_name=%q,branch="main",pod=%q}) or vector(0)`, - testNs, providerName, newControllerPod, + `{gitbranch_namespace=%q,gitbranch_name=%q,pod=%q}) or vector(0)`, + testNs, gitBranchName(providerName, "main"), newControllerPod, ), func(v float64) bool { return v == 0 }, "branch worker queue drained after restart", diff --git a/test/utils/utils.go b/test/utils/utils.go index a01068c7a..4f1c0dd0b 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -4,7 +4,6 @@ package utils import ( - "bufio" "bytes" "fmt" "io" @@ -83,53 +82,3 @@ func GetProjectDir() (string, error) { wd = strings.ReplaceAll(wd, "/test/e2e", "") return wd, nil } - -// UncommentCode searches for target in the file and remove the comment prefix -// of the target content. The target content may span multiple lines. -func UncommentCode(filename, target, prefix string) error { - // false positive - content, err := os.ReadFile(filename) - if err != nil { - return fmt.Errorf("failed to read file %q: %w", filename, err) - } - strContent := string(content) - - idx := strings.Index(strContent, target) - if idx < 0 { - return fmt.Errorf("unable to find the code %q to be uncomment", target) - } - - out := new(bytes.Buffer) - _, err = out.Write(content[:idx]) - if err != nil { - return fmt.Errorf("failed to write to output: %w", err) - } - - scanner := bufio.NewScanner(bytes.NewBufferString(target)) - if !scanner.Scan() { - return nil - } - for { - if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { - return fmt.Errorf("failed to write to output: %w", err) - } - // Avoid writing a newline in case the previous line was the last in target. - if !scanner.Scan() { - break - } - if _, err = out.WriteString("\n"); err != nil { - return fmt.Errorf("failed to write to output: %w", err) - } - } - - if _, err = out.Write(content[idx+len(target):]); err != nil { - return fmt.Errorf("failed to write to output: %w", err) - } - - //nolint:gosec - if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { - return fmt.Errorf("failed to write file %q: %w", filename, err) - } - - return nil -}