diff --git a/charts/substrate-atelet/README.md b/charts/substrate-atelet/README.md index 4f756e6b0b..60f774f654 100644 --- a/charts/substrate-atelet/README.md +++ b/charts/substrate-atelet/README.md @@ -44,6 +44,15 @@ This chart reads the same value paths as the substrate chart for `atelet`, `image`, `imagePullSecrets` and `global`, so the substrate release's values file can configure both releases. +These values are only in this chart: + +| Key | Default | Description | +|---|---|---| +| `atelet.extraVolumes` | `[]` | Volumes added to the atelet pod, such as a host file a credential provider reads | +| `atelet.extraVolumeMounts` | `[]` | Volume mounts added to the atelet container | +| `podMonitors.enabled` | `false` | Render a PodMonitor that scrapes atelet's `/metrics` | +| `podMonitors.apiVersion` | `monitoring.coreos.com/v1` | PodMonitor API version; `azmonitoring.coreos.com/v1` for Azure Managed Prometheus | + Object storage is set under `objectStore` instead of `rustfs` and `atelet.storageBackend`: diff --git a/charts/substrate-atelet/templates/atelet.yaml b/charts/substrate-atelet/templates/atelet.yaml index c8374bbf94..0d8f89aa22 100644 --- a/charts/substrate-atelet/templates/atelet.yaml +++ b/charts/substrate-atelet/templates/atelet.yaml @@ -308,6 +308,9 @@ spec: subPath: config.yaml {{- end }} readOnly: true +{{- end }} +{{- with .Values.atelet.extraVolumeMounts }} +{{- toYaml . | nindent 8 }} {{- end }} volumes: - name: objectstore-plugin @@ -368,3 +371,21 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem +{{- with .Values.atelet.extraVolumes }} +{{- toYaml . | nindent 6 }} +{{- end }} +{{- if .Values.podMonitors.enabled }} +--- +apiVersion: {{ .Values.podMonitors.apiVersion }} +kind: PodMonitor +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: atelet + podMetricsEndpoints: + - port: prometheus + path: /metrics +{{- end }} diff --git a/charts/substrate-atelet/values.yaml b/charts/substrate-atelet/values.yaml index b68515745e..684cee37c6 100644 --- a/charts/substrate-atelet/values.yaml +++ b/charts/substrate-atelet/values.yaml @@ -61,6 +61,10 @@ atelet: nodeSelector: {} extraArgs: [] extraEnv: [] + # Added to the atelet pod's volumes and its container's volumeMounts, such + # as a host file a credential provider reads. + extraVolumes: [] + extraVolumeMounts: [] resources: requests: cpu: 50m @@ -94,6 +98,13 @@ otel: enabled: true endpoint: "" +# A PodMonitor that scrapes atelet's /metrics endpoint. It needs the +# Prometheus Operator CRDs, or apiVersion azmonitoring.coreos.com/v1 for Azure +# Managed Prometheus. +podMonitors: + enabled: false + apiVersion: monitoring.coreos.com/v1 + # Values under `global` are visible to this chart and to every subchart. global: # -- Mirror registry that overrides where every image is pulled from. It wins diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 35d7bc96c8..79abefe720 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -89,6 +89,7 @@ See `values.yaml` for the full set; the important keys: | `atenetEgress.extraCredentialProviders` | `[]` | Credential providers you deploy, added after the bundled `k8s.io` provider, each `{name, address}`. `name` is the host of the `ate-secret://` URIs it serves; `address` is the `host:port` the egress gateway dials over mTLS. Its serving certificate must chain to the servicedns CA. See [egress credential injection](../../docs/guides/egress-credential-injection.md#the-plugin-api) | | `atenetEgress.agentgateway.logLevel` | `info` | Egress proxy `RUST_LOG` filter, e.g. `debug` or `info,agentgateway=debug` when troubleshooting | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `rustfs.storageClassName` | `""` | StorageClass of the RustFS data volume; empty uses the cluster's default | | `ateApi.snapshotLocation` | `""` | Cluster-wide snapshot URI; defaults to `gs:///` with bundled RustFS | | `atelet.enabled` | `true` | Deprecated. Set to `false` and install atelet with the [substrate-atelet](../substrate-atelet/README.md) chart | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | @@ -112,6 +113,8 @@ See `values.yaml` for the full set; the important keys: | `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` | | `otel.logs.enabled` | `true` | Enable OTLP actor events from ateapi and the ateworkers, plus the router access log. Actor events go to stdout when OTLP logs are disabled | | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | +| `podMonitors.enabled` | `false` | Render PodMonitors that scrape `/metrics` on ate-api-server, ate-controller, atenet-router and atenet-egress; ate-controller's only while `otel.metrics.enabled`, since it serves no `/metrics` otherwise | +| `podMonitors.apiVersion` | `monitoring.coreos.com/v1` | PodMonitor API version; `azmonitoring.coreos.com/v1` for Azure Managed Prometheus | ## Router timeouts diff --git a/charts/substrate/templates/podmonitors.yaml b/charts/substrate/templates/podmonitors.yaml new file mode 100644 index 0000000000..ebfe5d4c52 --- /dev/null +++ b/charts/substrate/templates/podmonitors.yaml @@ -0,0 +1,38 @@ +{{/* +Copyright 2026 The Agent Substrate Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.podMonitors.enabled }} +{{- $targets := list (list "ate-api-server" "prometheus") (list "atenet-router" "gw-metrics") (list "atenet-egress" "stats") }} +{{- /* ate-controller closes its metrics port when OTEL_METRICS_EXPORTER is none. */}} +{{- if .Values.otel.metrics.enabled }} +{{- $targets = append $targets (list "ate-controller" "metrics") }} +{{- end }} +{{- range $targets }} +--- +apiVersion: {{ $.Values.podMonitors.apiVersion }} +kind: PodMonitor +metadata: + name: {{ include "substrate.fullname" (list (index . 0) $) }} + namespace: {{ $.Release.Namespace }} +spec: + selector: + matchLabels: + app: {{ index . 0 }} + podMetricsEndpoints: + - port: {{ index . 1 }} + path: /metrics +{{- end }} +{{- end }} diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml index 95a5b10053..d8d39a6514 100644 --- a/charts/substrate/templates/rustfs.yaml +++ b/charts/substrate/templates/rustfs.yaml @@ -32,6 +32,9 @@ metadata: spec: accessModes: - ReadWriteOnce + {{- with .Values.rustfs.storageClassName }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.rustfs.storageSize }} diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index 87646b96fc..fb8e9abf0e 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -265,6 +265,25 @@ tests: value: - protocol: TCP port: api +- it: leaves the RustFS volume on the default StorageClass + template: rustfs.yaml + documentSelector: + path: kind + value: PersistentVolumeClaim + asserts: + - notExists: + path: spec.storageClassName +- it: puts the RustFS volume on rustfs.storageClassName + template: rustfs.yaml + documentSelector: + path: kind + value: PersistentVolumeClaim + set: + rustfs.storageClassName: managed-csi-premium + asserts: + - equal: + path: spec.storageClassName + value: managed-csi-premium - it: supplies the required API config and bootstraps the release client template: ate-api-server.yaml diff --git a/charts/substrate/tests/podmonitors_test.yaml b/charts/substrate/tests/podmonitors_test.yaml new file mode 100644 index 0000000000..b008145705 --- /dev/null +++ b/charts/substrate/tests/podmonitors_test.yaml @@ -0,0 +1,83 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +suite: PodMonitors +templates: +- podmonitors.yaml +release: + name: custom + namespace: custom-system +tests: +- it: renders no PodMonitors by default + template: podmonitors.yaml + asserts: + - hasDocuments: + count: 0 +- it: scrapes each control plane component's metrics port + template: podmonitors.yaml + set: + podMonitors: + enabled: true + apiVersion: azmonitoring.coreos.com/v1 + asserts: + - hasDocuments: + count: 4 + - isAPIVersion: + of: azmonitoring.coreos.com/v1 + - isKind: + of: PodMonitor + - equal: + path: metadata.namespace + value: custom-system + - equal: + path: metadata.name + value: custom-ate-api-server + documentIndex: 0 + - equal: + path: spec + value: + selector: + matchLabels: + app: ate-api-server + podMetricsEndpoints: + - port: prometheus + path: /metrics + documentIndex: 0 + - equal: + path: spec.podMetricsEndpoints[0].port + value: gw-metrics + documentIndex: 1 + - equal: + path: spec.podMetricsEndpoints[0].port + value: stats + documentIndex: 2 + - equal: + path: spec.selector.matchLabels.app + value: ate-controller + documentIndex: 3 + - equal: + path: spec.podMetricsEndpoints[0].port + value: metrics + documentIndex: 3 +- it: skips ate-controller when it serves no metrics + template: podmonitors.yaml + set: + podMonitors.enabled: true + otel.metrics.enabled: false + asserts: + - hasDocuments: + count: 3 + - notEqual: + path: spec.selector.matchLabels.app + value: ate-controller diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 5d0f7a4dce..e4bb553973 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -47,6 +47,8 @@ postgres: rustfs: enabled: true storageSize: 1Gi + # StorageClass of the data volume. Empty uses the cluster's default. + storageClassName: "" bucket: ate-snapshots accessKey: rustfsadmin secretKey: rustfsadmin @@ -229,6 +231,14 @@ otel: enabled: true endpoint: "" +# PodMonitors that scrape the control plane's /metrics endpoints. The +# substrate-atelet chart has its own for atelet. They need the Prometheus +# Operator CRDs, or apiVersion azmonitoring.coreos.com/v1 for Azure Managed +# Prometheus. +podMonitors: + enabled: false + apiVersion: monitoring.coreos.com/v1 + # Values under `global` are visible to this chart and to every subchart. A parent # chart or an operator sets one value here instead of one value per chart. global: diff --git a/cmd/atelet/helm_test.go b/cmd/atelet/helm_test.go index 63fce1ddf7..47b3dd69de 100644 --- a/cmd/atelet/helm_test.go +++ b/cmd/atelet/helm_test.go @@ -30,7 +30,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/utils/ptr" "sigs.k8s.io/yaml" ) @@ -145,6 +147,60 @@ providers: []` } } +func TestHelmAteletExtraVolumes(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is not installed") + } + objs := helmTemplate(t, "atelet", "../../charts/substrate-atelet", + "--set-json", `atelet.extraVolumes=[{"name": "extra", "hostPath": {"path": "/etc/extra.json", "type": "File"}}]`, + "--set-json", `atelet.extraVolumeMounts=[{"name": "extra", "mountPath": "/etc/extra.json", "readOnly": true}]`) + for _, obj := range objs { + if obj.GetKind() != "DaemonSet" { + continue + } + var ds appsv1.DaemonSet + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &ds); err != nil { + t.Fatal(err) + } + pod := ds.Spec.Template.Spec + wantVolume := corev1.Volume{Name: "extra", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/extra.json", Type: ptr.To(corev1.HostPathFile)}}} + if !slices.ContainsFunc(pod.Volumes, func(v corev1.Volume) bool { return reflect.DeepEqual(v, wantVolume) }) { + t.Errorf("atelet volumes %v don't include %v", pod.Volumes, wantVolume) + } + wantMount := corev1.VolumeMount{Name: "extra", MountPath: "/etc/extra.json", ReadOnly: true} + if !slices.ContainsFunc(pod.Containers[0].VolumeMounts, func(m corev1.VolumeMount) bool { return reflect.DeepEqual(m, wantMount) }) { + t.Errorf("atelet volume mounts %v don't include %v", pod.Containers[0].VolumeMounts, wantMount) + } + return + } + t.Fatal("no atelet DaemonSet rendered") +} + +func TestHelmAteletPodMonitor(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is not installed") + } + objs := helmTemplate(t, "atelet", "../../charts/substrate-atelet", + "--set", "podMonitors.enabled=true", "--set", "podMonitors.apiVersion=azmonitoring.coreos.com/v1") + for _, obj := range objs { + if obj.GetKind() != "PodMonitor" { + continue + } + if got := obj.GetAPIVersion(); got != "azmonitoring.coreos.com/v1" { + t.Errorf("PodMonitor apiVersion = %q, want azmonitoring.coreos.com/v1", got) + } + want := map[string]any{ + "selector": map[string]any{"matchLabels": map[string]any{"app": "atelet"}}, + "podMetricsEndpoints": []any{map[string]any{"port": "prometheus", "path": "/metrics"}}, + } + if diff := cmp.Diff(want, obj.Object["spec"]); diff != "" { + t.Errorf("PodMonitor spec mismatch (-want +got):\n%s", diff) + } + return + } + t.Fatal("no PodMonitor rendered") +} + func TestHelmSandboxAssetStorage(t *testing.T) { for _, tc := range []struct { name, backend string @@ -285,6 +341,8 @@ func TestHelmAteletChartMatchesSubstrateChart(t *testing.T) { "--set", "preview[0]=ALL", "--set", "otel.endpoint=http://collector:4317", "--set", "atelet.nodeSelector.pool=agents", + "--set", "atelet.imageCredentialProviderConfig=/var/lib/kubelet/credential-provider-config.yaml", + "--set", "atelet.imageCredentialProviderBinDir=/var/lib/kubelet/credential-provider", "--set", "image.tag=test", }, substrate: append([]string{"--set-json", `objectStorePlugin.extraEnv=[ diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go index 70d984cccd..46970175ff 100644 --- a/cmd/kubectl-ate/internal/cmd/install.go +++ b/cmd/kubectl-ate/internal/cmd/install.go @@ -34,6 +34,7 @@ import ( var ( configPath string + profile string chartDir string noTUI bool upgradeDryRun bool @@ -58,6 +59,9 @@ const configHelp = `--config takes a YAML file, or - for stdin: microvm: enabled: true assetBucket: my-assets # for s3 or gcs + imageRegistry: my.azurecr.io # pull every image from it + storageClass: fast # bundled volumes; empty for the default + otelEndpoint: http://otel-collector:4317 # your OTLP collector timeout: 10m # each wait for workloads releases: # Helm values all: # every release except substrate-crds @@ -71,7 +75,8 @@ const configHelp = `--config takes a YAML file, or - for stdin: substrate: # substrate values: otel: - endpoint: http://otel-collector:4317 + traces: + samplingRatio: 0.1 A release's own valuesFiles and values win over the ones in all.` @@ -94,9 +99,25 @@ described in charts/substrate/README.md and set postgres.type to external. To store snapshots in a bucket, set objectStore.type to s3 or gcs and set objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount. +storageClass sets the StorageClass of the bundled PostgreSQL and RustFS +volumes when they are created. + +imageRegistry pulls every image install deploys, including the bundled +PostgreSQL's, from that registry. It must hold the images under their usual +paths. + +otelEndpoint points atelet and the control plane at an OTLP collector you +already run, for traces, metrics and logs. Install doesn't deploy one. + +--profile names a config built into kubectl ate. The config from --config is +layered over it, so its settings win. Maps merge and lists replace, as in Helm +values. The aks profile sets storageClass to managed-csi-premium. + microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each node architecture, caches them under the user cache directory, uploads them to the object store, and applies the microvm SandboxConfig. Upgrade restages them. +Install also checks that a node advertises ate.dev/kvm, which atelet does on +nodes with /dev/kvm. ` + configHelp, Example: ` kubectl ate install -c install.yaml @@ -112,7 +133,7 @@ the object store, and applies the microvm SandboxConfig. Upgrade restages them. } defer closeTerminal(tty) if useTUI(cmd, tty) && configPath == "" { - if err := tui.Wizard(cmd.ErrOrStderr(), &opts.Config); err != nil { + if err := tui.Wizard(cmd.ErrOrStderr(), &opts.Config, profile == ""); err != nil { return err } } @@ -131,9 +152,9 @@ docs/reference/version-skew-policy.md upgrades after atelet. Upgrade doesn't reuse the values a release was installed with. Each release's values come from the chart and the config, the same way install -sets them, so pass the config you installed with, changed as needed. -It refuses downgrades, skipped minor releases, and CRD changes that would -strand stored objects. +sets them, so pass the config and --profile you installed with, changed as +needed. It refuses downgrades, skipped minor releases, and CRD changes that +would strand stored objects. --dry-run prints each release's manifest diff and changes nothing.`, Args: cobra.NoArgs, @@ -183,9 +204,9 @@ data.`, }, } -// options reads the config and finds the terminal to read keys and answers -// from. Without --config the config is install.DefaultConfig. The terminal is -// nil when there is none. Close it with closeTerminal. +// options reads the config, layered over --profile, and finds the terminal to +// read keys and answers from. Without --config the config is the profile's. +// The terminal is nil when there is none. Close it with closeTerminal. func options(cmd *cobra.Command) (install.Options, *os.File, error) { var opts install.Options if chartDir != "" { @@ -194,13 +215,15 @@ func options(cmd *cobra.Command) (install.Options, *os.File, error) { var err error switch configPath { case "": - opts.Config = install.DefaultConfig() + if opts.Config, err = install.ProfileConfig(profile); err != nil { + return opts, nil, err + } case "-": - opts.Config, err = install.ReadConfig(cmd.InOrStdin(), ".") + opts.Config, err = install.ReadConfig(cmd.InOrStdin(), ".", profile) default: var f *os.File if f, err = os.Open(configPath); err == nil { - opts.Config, err = install.ReadConfig(f, filepath.Dir(configPath)) + opts.Config, err = install.ReadConfig(f, filepath.Dir(configPath), profile) f.Close() } } @@ -339,6 +362,7 @@ func init() { for _, c := range []*cobra.Command{installCmd, upgradeCmd, uninstallCmd} { c.Flags().StringVarP(&configPath, "config", "c", "", "Install config file, or - for stdin") c.Flags().BoolVar(&noTUI, "no-tui", false, "Print plain progress lines instead of the interactive view") + c.Flags().StringVar(&profile, "profile", "", fmt.Sprintf("Built-in config to layer the config over, one of %v", install.Profiles())) } for _, c := range []*cobra.Command{installCmd, upgradeCmd} { c.Flags().StringVar(&chartDir, "chart-dir", "", "Load charts from this directory instead of the ones built into kubectl ate") diff --git a/pkg/kagent/install/config.go b/pkg/kagent/install/config.go index d45387c29f..bfeb21dd9a 100644 --- a/pkg/kagent/install/config.go +++ b/pkg/kagent/install/config.go @@ -16,25 +16,45 @@ package install import ( "bytes" + "embed" "encoding/json" "errors" "fmt" "io" + "net/url" "path/filepath" "strings" "time" + "unicode" "helm.sh/helm/v4/pkg/cli/values" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" "sigs.k8s.io/yaml" ) +// profileFS holds the built-in profiles, one config file each. +// +//go:embed profiles/*.yaml +var profileFS embed.FS + // Config describes an installation. Install, upgrade and uninstall read it // from the file passed to --config. type Config struct { Postgres Postgres `json:"postgres,omitzero"` ObjectStore ObjectStore `json:"objectStore,omitzero"` MicroVM MicroVM `json:"microvm,omitzero"` + // ImageRegistry is a registry to pull every image kubectl ate installs + // from, like myregistry.azurecr.io. It must hold the images under their + // usual paths. Empty keeps each image's own registry. + ImageRegistry string `json:"imageRegistry,omitempty"` + // StorageClass is the StorageClass of the bundled PostgreSQL and RustFS + // volumes. Empty uses the cluster's default. + StorageClass string `json:"storageClass,omitempty"` + // OTelEndpoint is an existing OTLP collector that atelet and the control + // plane export traces, metrics and logs to. Empty leaves the charts' otel + // values as they are. + OTelEndpoint string `json:"otelEndpoint,omitempty"` // Timeout bounds each wait for workloads to become ready. Timeout metav1.Duration `json:"timeout,omitzero"` // Releases holds Helm values for the releases. @@ -181,20 +201,39 @@ func DefaultConfig() Config { } } -// ReadConfig decodes a config from r and fills unset fields from -// DefaultConfig. dir resolves relative ValuesFiles. -func ReadConfig(r io.Reader, dir string) (Config, error) { +// Profiles lists the built-in profiles. Each is a config file that ReadConfig +// and ProfileConfig layer a config over. +func Profiles() []string { + entries, err := profileFS.ReadDir("profiles") + if err != nil { + panic(err) + } + var names []string + for _, e := range entries { + names = append(names, strings.TrimSuffix(e.Name(), ".yaml")) + } + return names +} + +// ReadConfig decodes a config from r and fills unset fields from the profile, +// then from DefaultConfig. An empty profile is none. dir resolves relative +// ValuesFiles. +func ReadConfig(r io.Reader, dir string, profile string) (Config, error) { data, err := io.ReadAll(r) if err != nil { return Config{}, err } - cfg := DefaultConfig() if len(bytes.TrimSpace(data)) == 0 { return Config{}, errors.New("config is empty") } - if err := yaml.UnmarshalStrict(data, &cfg); err != nil { + var user map[string]any + if err := yaml.UnmarshalStrict(data, &user); err != nil { return Config{}, fmt.Errorf("while decoding config: %w", err) } + cfg, err := layerConfig(profile, user) + if err != nil { + return Config{}, err + } for _, rv := range cfg.Releases.all() { for i, f := range rv.ValuesFiles { if !filepath.IsAbs(f) { @@ -205,7 +244,80 @@ func ReadConfig(r io.Reader, dir string) (Config, error) { return cfg, cfg.validate() } +// ProfileConfig returns DefaultConfig with the profile's settings. An empty +// profile is DefaultConfig. +func ProfileConfig(profile string) (Config, error) { + cfg, err := layerConfig(profile, nil) + if err != nil { + return Config{}, err + } + return cfg, cfg.validate() +} + +// layerConfig decodes user over the profile over DefaultConfig. Maps merge +// and lists replace, as in Helm values. +func layerConfig(profile string, user map[string]any) (Config, error) { + var base map[string]any + if profile != "" { + data, err := profileFS.ReadFile("profiles/" + profile + ".yaml") + if err != nil { + return Config{}, fmt.Errorf("invalid profile %q: must be one of %v", profile, Profiles()) + } + if err := yaml.UnmarshalStrict(data, &base); err != nil { + return Config{}, fmt.Errorf("while decoding profile %s: %w", profile, err) + } + } + data, err := json.Marshal(mergeValues(base, user)) + if err != nil { + return Config{}, err + } + cfg := DefaultConfig() + if err := decodeStrict(data, &cfg); err != nil { + return Config{}, fmt.Errorf("while decoding config: %w", err) + } + return cfg, nil +} + +// validateImageRegistry checks that s is empty or a registry host. +func validateImageRegistry(s string) error { + if s == "" { + return nil + } + // Image references read a first component without a dot or a port as a + // Docker Hub namespace. + if !strings.ContainsAny(s, ".:") || strings.ContainsFunc(s, func(r rune) bool { return r == '/' || unicode.IsSpace(r) }) { + return fmt.Errorf("invalid imageRegistry %q: must be a registry host like myregistry.azurecr.io", s) + } + return nil +} + +// ValidateStorageClass checks that s is empty or a valid StorageClass name. +func ValidateStorageClass(s string) error { + if s == "" { + return nil + } + if errs := validation.IsDNS1123Subdomain(s); len(errs) > 0 { + return fmt.Errorf("invalid StorageClass name %q: %s", s, strings.Join(errs, "; ")) + } + return nil +} + +// ValidateOTelEndpoint checks that s is empty or an http or https URL. +func ValidateOTelEndpoint(s string) error { + if s == "" { + return nil + } + u, err := url.Parse(s) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return fmt.Errorf("invalid OTLP endpoint %q: must be a URL like http://otel-collector:4317", s) + } + return nil +} + func (c Config) validate() error { + if err := validateImageRegistry(c.ImageRegistry); err != nil { + return err + } switch c.Postgres.Type { case PostgresBundled, PostgresExternal: default: @@ -214,6 +326,12 @@ func (c Config) validate() error { if err := c.ObjectStore.validate(); err != nil { return err } + if err := ValidateStorageClass(c.StorageClass); err != nil { + return fmt.Errorf("storageClass: %w", err) + } + if err := ValidateOTelEndpoint(c.OTelEndpoint); err != nil { + return fmt.Errorf("otelEndpoint: %w", err) + } if c.MicroVM.Enabled && c.ObjectStore.Type != ObjectStoreBundled && c.MicroVM.AssetBucket == "" { return errors.New("microvm with an s3 or gcs objectStore needs microvm.assetBucket") } @@ -223,12 +341,30 @@ func (c Config) validate() error { return nil } +// registryValues returns the Helm values that pull a release's images from +// ImageRegistry. +func (c Config) registryValues() map[string]any { + if c.ImageRegistry == "" { + return nil + } + return map[string]any{"global": map[string]any{"imageRegistry": c.ImageRegistry}} +} + func (r *Releases) all() []*ReleaseValues { return []*ReleaseValues{&r.All, &r.PodCert, &r.Atelet, &r.Substrate} } -// userValues returns each release's values, keyed by release name: All's -// files and values, then the release's own. +// otelValues returns the Helm values that point r at OTelEndpoint. +func (c Config) otelValues(r release) map[string]any { + if c.OTelEndpoint == "" || (r != ateletRelease && r != substrateRelease) { + return nil + } + return map[string]any{"otel": map[string]any{"endpoint": c.OTelEndpoint}} +} + +// userValues returns each release's values, keyed by release name: +// ImageRegistry's and OTelEndpoint's, All's files and values, then the +// release's own. func (c Config) userValues() (map[string]map[string]any, error) { all, err := c.Releases.All.read() if err != nil { @@ -244,7 +380,7 @@ func (c Config) userValues() (map[string]map[string]any, error) { if err != nil { return nil, err } - user[r.name] = mergeValues(all, own) + user[r.name] = mergeValues(c.registryValues(), c.otelValues(r), all, own) } return user, nil } diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go index d3094e3fab..32128c125f 100644 --- a/pkg/kagent/install/config_test.go +++ b/pkg/kagent/install/config_test.go @@ -34,6 +34,7 @@ func TestReadConfig(t *testing.T) { tests := []struct { name string in string + profile string want Config wantErr string }{ @@ -67,7 +68,29 @@ releases: c.Releases.Substrate.ValuesFiles = []string{"/cfg/db.yaml"} }), }, + {name: "storage class", in: "storageClass: fast\n", want: withDefaults(func(c *Config) { c.StorageClass = "fast" })}, + {name: "bad storage class", in: "storageClass: Premium_SSD\n", wantErr: "invalid StorageClass name"}, + {name: "profile", in: "{}", profile: "aks", want: withDefaults(func(c *Config) { c.StorageClass = "managed-csi-premium" })}, + {name: "config wins over the profile", in: "storageClass: fast\n", profile: "aks", want: withDefaults(func(c *Config) { c.StorageClass = "fast" })}, + { + name: "profile without bundled volumes", + in: "postgres: external\nobjectStore: s3\n", + profile: "aks", + want: withDefaults(func(c *Config) { + c.Postgres, c.ObjectStore = Postgres{Type: PostgresExternal}, ObjectStore{Type: ObjectStoreS3} + c.StorageClass = "managed-csi-premium" + }), + }, {name: "empty", in: "", wantErr: "config is empty"}, + {name: "bad profile", in: "{}", profile: "eks", wantErr: "invalid profile"}, + {name: "profile in the config", in: "profile: aks\n", wantErr: "profile"}, + {name: "image registry", in: "imageRegistry: r.azurecr.io\n", want: withDefaults(func(c *Config) { c.ImageRegistry = "r.azurecr.io" })}, + {name: "image registry with port", in: "imageRegistry: localhost:5001\n", want: withDefaults(func(c *Config) { c.ImageRegistry = "localhost:5001" })}, + {name: "image registry with path", in: "imageRegistry: r.azurecr.io/substrate\n", wantErr: "invalid imageRegistry"}, + {name: "image registry without domain", in: "imageRegistry: myregistry\n", wantErr: "invalid imageRegistry"}, + {name: "image registry with space", in: "imageRegistry: 'r.azurecr.io '\n", wantErr: "invalid imageRegistry"}, + {name: "otel endpoint", in: "otelEndpoint: http://collector:4317\n", want: withDefaults(func(c *Config) { c.OTelEndpoint = "http://collector:4317" })}, + {name: "otel endpoint without scheme", in: "otelEndpoint: collector:4317\n", wantErr: "invalid OTLP endpoint"}, {name: "unknown field", in: "postgress: external\n", wantErr: "postgress"}, {name: "unknown release", in: "releases: {crds: {values: {a: 1}}}\n", wantErr: "crds"}, {name: "bad postgres", in: "postgres: cloud\n", wantErr: "invalid postgres"}, @@ -104,7 +127,7 @@ releases: } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, err := ReadConfig(strings.NewReader(tc.in), "/cfg") + got, err := ReadConfig(strings.NewReader(tc.in), "/cfg", tc.profile) if tc.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tc.wantErr) { t.Fatalf("ReadConfig() error = %v, want it to contain %q", err, tc.wantErr) @@ -157,3 +180,45 @@ func TestConfigUserValues(t *testing.T) { t.Errorf("userValues() mismatch (-want +got):\n%s", diff) } } + +func TestProfileConfig(t *testing.T) { + for _, p := range append(Profiles(), "") { + if _, err := ProfileConfig(p); err != nil { + t.Errorf("ProfileConfig(%q): %v", p, err) + } + } + if _, err := ProfileConfig("eks"); err == nil || !strings.Contains(err.Error(), "invalid profile") { + t.Errorf("ProfileConfig(eks) error = %v, want invalid profile", err) + } +} + +func TestConfigUserValuesRegistryAndOTel(t *testing.T) { + registry := func(r string) map[string]any { + return map[string]any{"global": map[string]any{"imageRegistry": r}} + } + otel := func(endpoint string) map[string]any { + return map[string]any{"endpoint": endpoint} + } + substrateValues := registry("mirror.example.com") + substrateValues["otel"] = otel("http://other:4317") + c := Config{ + ImageRegistry: "r.azurecr.io", + OTelEndpoint: "http://collector:4317", + Releases: Releases{Substrate: ReleaseValues{Values: substrateValues}}, + } + got, err := c.userValues() + if err != nil { + t.Fatal(err) + } + // The release's own values win, and podcert gets no otel. + atelet := registry("r.azurecr.io") + atelet["otel"] = otel("http://collector:4317") + want := map[string]map[string]any{ + "substrate-podcert": registry("r.azurecr.io"), + "substrate-atelet": atelet, + "substrate": substrateValues, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("userValues() mismatch (-want +got):\n%s", diff) + } +} diff --git a/pkg/kagent/install/install_test.go b/pkg/kagent/install/install_test.go index b80d9cc2f9..3310bae9b2 100644 --- a/pkg/kagent/install/install_test.go +++ b/pkg/kagent/install/install_test.go @@ -25,11 +25,16 @@ import ( "testing" chart "helm.sh/helm/v4/pkg/chart/v2" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" chartsfs "go.ate.dev/substrate/charts" + "go.ate.dev/substrate/internal/deviceplugin" ) func TestMergeValues(t *testing.T) { @@ -185,7 +190,7 @@ func secretData(t *testing.T, kc *fake.Clientset) map[string]any { func TestApplyPostgresManifest(t *testing.T) { ctx := context.Background() - manifest, err := postgresManifest() + manifest, err := postgresManifest(Config{}) if err != nil { t.Fatal(err) } @@ -209,6 +214,156 @@ func TestApplyPostgresManifest(t *testing.T) { } } +func postgresStatefulSetFor(t *testing.T, c Config) *appsv1.StatefulSet { + t.Helper() + ctx := context.Background() + manifest, err := postgresManifest(c) + if err != nil { + t.Fatal(err) + } + kc := fake.NewClientset() + if err := applyManifest(ctx, kc, manifest); err != nil { + t.Fatal(err) + } + ss, err := kc.AppsV1().StatefulSets(SystemNamespace).Get(ctx, postgresStatefulSet, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return ss +} + +func TestPostgresManifestRegistry(t *testing.T) { + images := func(c Config) []string { + t.Helper() + ss := postgresStatefulSetFor(t, c) + var out []string + for _, c := range append(ss.Spec.Template.Spec.InitContainers, ss.Spec.Template.Spec.Containers...) { + out = append(out, c.Image) + } + return out + } + base, mirrored := images(Config{}), images(Config{ImageRegistry: "r.azurecr.io"}) + if len(base) == 0 || len(base) != len(mirrored) { + t.Fatalf("got images %v and %v, want the same nonzero count", base, mirrored) + } + // The digest stays, so the mirror must hold the same image. + for i := range base { + if want := "r.azurecr.io/" + base[i]; mirrored[i] != want { + t.Errorf("image %d = %s, want %s", i, mirrored[i], want) + } + } +} + +func TestPostgresManifestStorageClass(t *testing.T) { + for _, sc := range []string{"", "managed-csi-premium"} { + ss := postgresStatefulSetFor(t, Config{StorageClass: sc}) + vct := ss.Spec.VolumeClaimTemplates[0].Spec + if got := ptr.Deref(vct.StorageClassName, ""); got != sc { + t.Errorf("StorageClass %q: storageClassName = %q", sc, got) + } + // The overlay keeps the kind overlay's patches. + if got := vct.Resources.Requests.Storage().String(); got != "1Gi" { + t.Errorf("StorageClass %q: volume size = %s, want 1Gi", sc, got) + } + } +} + +func TestPostgresStorageClass(t *testing.T) { + for _, tc := range []struct { + name string + existing *string + want string + }{ + {name: "new install uses the config's", want: "managed-csi-premium"}, + {name: "existing keeps the default", existing: ptr.To(""), want: ""}, + {name: "existing keeps its own", existing: ptr.To("managed-csi"), want: "managed-csi"}, + } { + t.Run(tc.name, func(t *testing.T) { + kc := fake.NewClientset() + if tc.existing != nil { + ss := postgresStatefulSetFor(t, Config{StorageClass: *tc.existing}) + if _, err := kc.AppsV1().StatefulSets(SystemNamespace).Create(t.Context(), ss, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + got, err := postgresStorageClass(t.Context(), kc, "managed-csi-premium") + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("postgresStorageClass() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestSubstrateValuesStorageClass(t *testing.T) { + kc := fake.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: SystemNamespace, Name: rustfsCredentialsSecret}, + Data: map[string][]byte{"AWS_ACCESS_KEY_ID": []byte("a"), "AWS_SECRET_ACCESS_KEY": []byte("s")}, + }) + in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreBundled}, Postgres: Postgres{Type: PostgresBundled}, StorageClass: "fast"}}} + defaults, _, err := in.substrateValues(context.Background()) + if err != nil { + t.Fatal(err) + } + want := map[string]any{"rustfs": map[string]any{"storageClassName": "fast"}} + if !reflect.DeepEqual(defaults, want) { + t.Errorf("substrate defaults = %v, want %v", defaults, want) + } +} + +func TestCheckKVM(t *testing.T) { + node := func(name string, kvm bool) runtime.Object { + n := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}} + if kvm { + n.Status.Allocatable = corev1.ResourceList{deviceplugin.ResourceKVM: resource.MustParse("4096")} + } + return n + } + for _, tc := range []struct { + name string + nodes []runtime.Object + wantErr bool + }{ + {name: "one node", nodes: []runtime.Object{node("a", false), node("b", true)}}, + {name: "none", nodes: []runtime.Object{node("a", false)}, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + in := &Installer{kube: fake.NewClientset(tc.nodes...)} + // The first check runs before the poll sees the canceled context. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := in.checkKVM(ctx, func(string) {}) + if gotErr := err != nil; gotErr != tc.wantErr { + t.Errorf("checkKVM() error = %v, want error %t", err, tc.wantErr) + } + }) + } +} + +func TestCheckPodCertificateAPI(t *testing.T) { + pcr := []metav1.APIResource{{Name: "podcertificaterequests"}} + for _, tc := range []struct { + name string + resources []*metav1.APIResourceList + wantErr bool + }{ + {name: "v1", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1", APIResources: pcr}}}, + {name: "v1beta1", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1beta1", APIResources: pcr}}}, + {name: "v1 without podcertificaterequests", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1", APIResources: []metav1.APIResource{{Name: "clustertrustbundles"}}}}, wantErr: true}, + {name: "neither", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + kc := fake.NewClientset() + kc.Resources = tc.resources + if err := checkPodCertificateAPI(&Installer{kube: kc}); (err != nil) != tc.wantErr { + t.Errorf("checkPodCertificateAPI() = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} + func TestCheckExternalObjectStore(t *testing.T) { s3 := ObjectStore{Type: ObjectStoreS3, Bucket: "b"} gcs := ObjectStore{Type: ObjectStoreGCS, Bucket: "b", GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}} diff --git a/pkg/kagent/install/installer.go b/pkg/kagent/install/installer.go index 393495a0b0..f0b0556c2b 100644 --- a/pkg/kagent/install/installer.go +++ b/pkg/kagent/install/installer.go @@ -200,6 +200,9 @@ func (in *Installer) InstallSteps() ([]Step, error) { {"Configure object storage", in.configureObjectStore}, {"Install atelet", in.helmStep(ateletRelease, in.ateletValues)}, } + if in.opts.MicroVM.Enabled { + steps = append(steps, Step{"Check KVM", in.checkKVM}) + } if in.opts.Postgres.Type == PostgresBundled { steps = append(steps, Step{"Install PostgreSQL", in.applyPostgres}, diff --git a/pkg/kagent/install/microvm.go b/pkg/kagent/install/microvm.go index b4d13dc24c..b278d0a2eb 100644 --- a/pkg/kagent/install/microvm.go +++ b/pkg/kagent/install/microvm.go @@ -24,6 +24,7 @@ import ( "path/filepath" "slices" "strings" + "time" "cloud.google.com/go/storage" "github.com/aws/aws-sdk-go-v2/aws" @@ -34,9 +35,11 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/yaml" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" + "go.ate.dev/substrate/internal/deviceplugin" "go.ate.dev/substrate/internal/portforward" ateinstall "go.ate.dev/substrate/manifests/ate-install" "go.ate.dev/substrate/pkg/kagent/microvmassets" @@ -344,3 +347,25 @@ func uploadFile(ctx context.Context, b assetBucket, object, p string) error { defer f.Close() return b.put(ctx, object, f) } + +// checkKVM waits for a node to advertise ate.dev/kvm, which microVM workers +// request. atelet advertises it on the nodes it runs on that have /dev/kvm. +func (in *Installer) checkKVM(ctx context.Context, progress func(string)) (string, error) { + err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + nodes, err := in.kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + for _, n := range nodes.Items { + if q, ok := n.Status.Allocatable[deviceplugin.ResourceKVM]; ok && !q.IsZero() { + return true, nil + } + } + progress(fmt.Sprintf("waiting for %s on %d nodes", deviceplugin.ResourceKVM, len(nodes.Items))) + return false, nil + }) + if err != nil { + return "", fmt.Errorf("no node advertises %s, so microVM workers can't schedule; atelet's nodes need /dev/kvm: %w", deviceplugin.ResourceKVM, err) + } + return deviceplugin.ResourceKVM + " advertised", nil +} diff --git a/pkg/kagent/install/postgres.go b/pkg/kagent/install/postgres.go index 8cc002de24..aa744368d0 100644 --- a/pkg/kagent/install/postgres.go +++ b/pkg/kagent/install/postgres.go @@ -26,6 +26,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" @@ -34,9 +35,12 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" + "k8s.io/utils/ptr" "sigs.k8s.io/kustomize/api/krusty" kusttypes "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/kustomize/kyaml/resid" + "sigs.k8s.io/yaml" ateinstall "go.ate.dev/substrate/manifests/ate-install" "go.ate.dev/substrate/pkg/postgressetup" @@ -57,7 +61,13 @@ const ( // applyPostgres applies manifests/ate-install/kind/postgres and waits for it. // That overlay sizes the database for evaluation rather than production. func (in *Installer) applyPostgres(ctx context.Context, progress func(string)) (string, error) { - manifest, err := postgresManifest() + c := in.opts.Config + sc, err := postgresStorageClass(ctx, in.kube, c.StorageClass) + if err != nil { + return "", err + } + c.StorageClass = sc + manifest, err := postgresManifest(c) if err != nil { return "", err } @@ -71,7 +81,24 @@ func (in *Installer) applyPostgres(ctx context.Context, progress func(string)) ( return "statefulset/" + postgresStatefulSet + " ready", nil } -func postgresManifest() ([]byte, error) { +// postgresStorageClass returns the StorageClass to apply the bundled +// PostgreSQL with. An existing StatefulSet keeps its own, since its +// volumeClaimTemplates can't change. +func postgresStorageClass(ctx context.Context, kc kubernetes.Interface, sc string) (string, error) { + ss, err := kc.AppsV1().StatefulSets(SystemNamespace).Get(ctx, postgresStatefulSet, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return sc, nil + } + if err != nil { + return "", err + } + return ptr.Deref(ss.Spec.VolumeClaimTemplates[0].Spec.StorageClassName, ""), nil +} + +// postgresManifest builds the bundled PostgreSQL manifest. c's image registry +// replaces the image's, the way the charts' global.imageRegistry does, and c's +// StorageClass sets the volume's. +func postgresManifest(c Config) ([]byte, error) { mem := filesys.MakeFsInMemory() err := fs.WalkDir(ateinstall.Postgres, ".", func(p string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { @@ -86,10 +113,42 @@ func postgresManifest() ([]byte, error) { if err != nil { return nil, err } + dir := "/kind/postgres" + overlay := kusttypes.Kustomization{ + TypeMeta: kusttypes.TypeMeta{APIVersion: kusttypes.KustomizationVersion, Kind: kusttypes.KustomizationKind}, + Resources: []string{"../kind/postgres"}, + } + if r := c.ImageRegistry; r != "" { + overlay.Images = []kusttypes.Image{{Name: "postgres", NewName: r + "/postgres"}} + } + if sc := c.StorageClass; sc != "" { + patch, err := yaml.Marshal([]map[string]any{{ + "op": "add", + "path": "/spec/volumeClaimTemplates/0/spec/storageClassName", + "value": sc, + }}) + if err != nil { + return nil, err + } + overlay.Patches = []kusttypes.Patch{{ + Target: &kusttypes.Selector{ResId: resid.NewResIdWithNamespace(resid.NewGvk("apps", "v1", "StatefulSet"), postgresStatefulSet, SystemNamespace)}, + Patch: string(patch), + }} + } + if overlay.Images != nil || overlay.Patches != nil { + k, err := yaml.Marshal(overlay) + if err != nil { + return nil, err + } + dir = "/overlay" + if err := mem.WriteFile(dir+"/kustomization.yaml", k); err != nil { + return nil, err + } + } opts := krusty.MakeDefaultOptions() // The overlay references its base outside its own directory. opts.LoadRestrictions = kusttypes.LoadRestrictionsNone - res, err := krusty.MakeKustomizer(opts).Run(mem, "/kind/postgres") + res, err := krusty.MakeKustomizer(opts).Run(mem, dir) if err != nil { return nil, fmt.Errorf("while building the PostgreSQL manifest: %w", err) } diff --git a/pkg/kagent/install/profiles/aks.yaml b/pkg/kagent/install/profiles/aks.yaml new file mode 100644 index 0000000000..7c43f719af --- /dev/null +++ b/pkg/kagent/install/profiles/aks.yaml @@ -0,0 +1,16 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Azure Kubernetes Service. The bundled volumes use Premium SSDs. +storageClass: managed-csi-premium diff --git a/pkg/kagent/install/steps.go b/pkg/kagent/install/steps.go index 521deca08c..894d5ce42d 100644 --- a/pkg/kagent/install/steps.go +++ b/pkg/kagent/install/steps.go @@ -16,6 +16,7 @@ package install import ( "context" + "errors" "fmt" "time" @@ -73,21 +74,23 @@ func (in *Installer) preflight(ctx context.Context, progress func(string)) (stri } // checkPodCertificateAPI checks that the cluster serves PodCertificateRequest, -// which the certificate controller signs. +// which the certificate controller signs, at v1 or v1beta1. func checkPodCertificateAPI(in *Installer) error { - const gv = "certificates.k8s.io/v1beta1" - res, err := in.kube.Discovery().ServerResourcesForGroupVersion(gv) - if err == nil { + for _, gv := range []string{"certificates.k8s.io/v1", "certificates.k8s.io/v1beta1"} { + res, err := in.kube.Discovery().ServerResourcesForGroupVersion(gv) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return fmt.Errorf("while discovering %s: %w", gv, err) + } for _, r := range res.APIResources { if r.Name == "podcertificaterequests" { return nil } } } - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("while discovering %s: %w", gv, err) - } - return fmt.Errorf("cluster doesn't serve %s podcertificaterequests; enable the PodCertificateRequest feature gate and API", gv) + return errors.New("cluster serves neither certificates.k8s.io/v1 nor v1beta1 podcertificaterequests; enable the PodCertificateRequest feature gate and API") } // valuesFunc returns installer defaults, which user values override, and @@ -184,12 +187,16 @@ func (in *Installer) substrateValues(ctx context.Context) (map[string]any, map[s // atelet is its own release so it can upgrade first. "atelet": map[string]any{"enabled": false}, } + var defaults map[string]any if in.opts.ObjectStore.Type == ObjectStoreBundled { access, secret, err := in.rustfsCredentials(ctx) if err != nil { return nil, nil, err } forced["rustfs"] = map[string]any{"accessKey": access, "secretKey": secret} + if sc := in.opts.StorageClass; sc != "" { + defaults = map[string]any{"rustfs": map[string]any{"storageClassName": sc}} + } } else { _, substrate := in.externalStoreValues() forced = mergeValues(forced, substrate) @@ -197,7 +204,7 @@ func (in *Installer) substrateValues(ctx context.Context) (map[string]any, map[s if in.opts.Postgres.Type == PostgresBundled { forced["postgres"] = map[string]any{"clientCertificates": map[string]any{"enabled": true}} } - return nil, forced, nil + return defaults, forced, nil } // externalStoreValues returns the values Config.ObjectStore sets on the diff --git a/pkg/kagent/install/tui/tui_test.go b/pkg/kagent/install/tui/tui_test.go index 84ca770fdd..e8dbdd731a 100644 --- a/pkg/kagent/install/tui/tui_test.go +++ b/pkg/kagent/install/tui/tui_test.go @@ -78,6 +78,12 @@ func TestHint(t *testing.T) { c.Releases.All.ValuesFiles = []string{"/etc/gke.yaml"} return c }(), + func() install.Config { + c := install.DefaultConfig() + c.ImageRegistry, c.StorageClass = "r.azurecr.io", "managed-csi-premium" + c.OTelEndpoint = "http://collector:4317" + return c + }(), } { hint, err := Hint(&cfg) if err != nil { @@ -87,7 +93,7 @@ func TestHint(t *testing.T) { if !ok { t.Fatalf("Hint() has no config:\n%s", hint) } - got, err := install.ReadConfig(strings.NewReader(doc), "") + got, err := install.ReadConfig(strings.NewReader(doc), "", "") if err != nil { t.Fatalf("ReadConfig(Hint()): %v\n%s", err, doc) } diff --git a/pkg/kagent/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go index d169513f73..6f7455450b 100644 --- a/pkg/kagent/install/tui/wizard.go +++ b/pkg/kagent/install/tui/wizard.go @@ -27,11 +27,29 @@ import ( ) // Wizard asks for the install choices, stores them in cfg, and prints the -// matching config to w. It returns install.ErrAborted if the user declines or -// quits. -func Wizard(w io.Writer, cfg *install.Config) error { +// matching config to w. askProfile first asks for a profile and replaces cfg +// with install.ProfileConfig of the answer. It returns install.ErrAborted if +// the user declines or quits. +func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error { + if askProfile { + var profile string + options := []huh.Option[string]{huh.NewOption("Any Kubernetes cluster", "")} + for _, p := range install.Profiles() { + options = append(options, huh.NewOption(p, p)) + } + if err := runForm(huh.NewForm(huh.NewGroup( + huh.NewSelect[string]().Title("Profile").Options(options...).Value(&profile), + ))); err != nil { + return err + } + var err error + if *cfg, err = install.ProfileConfig(profile); err != nil { + return err + } + } store := &cfg.ObjectStore external := func() bool { return store.Type != install.ObjectStoreBundled } + bundledVolumes := func() bool { return cfg.Postgres.Type == install.PostgresBundled || !external() } confirmed := true form := huh.NewForm( huh.NewGroup( @@ -82,6 +100,13 @@ func Wizard(w io.Writer, cfg *install.Config) error { Value(&store.GCS.TransferServiceAccount). Validate(notEmpty), ).WithHideFunc(func() bool { return store.Type != install.ObjectStoreGCS }), + huh.NewGroup( + huh.NewInput(). + Title("StorageClass"). + Description("For the bundled volumes. Leave empty for the cluster's default."). + Value(&cfg.StorageClass). + Validate(install.ValidateStorageClass), + ).WithHideFunc(func() bool { return !bundledVolumes() }), huh.NewGroup( huh.NewInput(). Title("MicroVM asset bucket"). @@ -89,6 +114,13 @@ func Wizard(w io.Writer, cfg *install.Config) error { Value(&cfg.MicroVM.AssetBucket). Validate(notEmpty), ).WithHideFunc(func() bool { return !external() || !cfg.MicroVM.Enabled }), + huh.NewGroup( + huh.NewInput(). + Title("OTLP endpoint"). + Description("An OTLP collector you already run, for traces, metrics and logs, like\nhttp://otel-collector.observability:4317. Install doesn't deploy one. Leave empty for none."). + Value(&cfg.OTelEndpoint). + Validate(install.ValidateOTelEndpoint), + ), huh.NewGroup( huh.NewConfirm(). Title("Install Agent Substrate?"). @@ -97,10 +129,7 @@ func Wizard(w io.Writer, cfg *install.Config) error { Value(&confirmed), ), ) - if err := form.Run(); err != nil { - if errors.Is(err, huh.ErrUserAborted) { - return install.ErrAborted - } + if err := runForm(form); err != nil { return err } if !confirmed { @@ -136,6 +165,15 @@ func Hint(cfg *install.Config) (string, error) { return "To skip these questions next time, save this as install.yaml and run kubectl ate install -c install.yaml:\n\n" + string(data) + "\n", nil } +// runForm runs f. It returns install.ErrAborted if the user quits. +func runForm(f *huh.Form) error { + err := f.Run() + if errors.Is(err, huh.ErrUserAborted) { + return install.ErrAborted + } + return err +} + func notEmpty(s string) error { if strings.TrimSpace(s) == "" { return errors.New("required") diff --git a/pkg/kagent/install/uninstall.go b/pkg/kagent/install/uninstall.go index 604916271c..3a2bb78859 100644 --- a/pkg/kagent/install/uninstall.go +++ b/pkg/kagent/install/uninstall.go @@ -224,7 +224,7 @@ func (in *Installer) uninstallStep(r release) func(context.Context, func(string) // removePostgres deletes the bundled PostgreSQL objects that install applied, // identified by kubectl ate's field manager. Its volume stays until --purge. func (in *Installer) removePostgres(ctx context.Context, _ func(string)) (string, error) { - manifest, err := postgresManifest() + manifest, err := postgresManifest(Config{}) if err != nil { return "", err }