From ec10406e0b4a4cac72d5e547db4c0553e1ae58b9 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:40:38 -0700 Subject: [PATCH] Allow running the parquet converter as a StatefulSet The parquet converter downloads each block into its data_dir before it converts it, and it wipes /compact before each download, so replicas must not share a volume. As a Deployment it has no volume of its own: the container runs with readOnlyRootFilesystem, so the default data_dir (./data) cannot be written unless extraVolumes adds one. Add parquet_converter.statefulSet.enabled, the same switch as ingester.statefulSet.enabled. When it is on, the converter is a StatefulSet with one Persistent Volume Claim per pod from parquet_converter.persistentVolume (or an emptyDir when that is off), mounted at /data. The image's working directory is /, so the default data_dir ./data is on that volume with no config change. The claim keeps the same volume across restarts and rollouts, and persistentVolume.retentionPolicy controls the claims on scale-down and on deletion, as for the ingester, store-gateway and compactor. The default stays a Deployment, so nothing changes for existing users. Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- CHANGELOG.md | 1 + README.md | 10 ++ .../parquet-converter-dep.yaml | 2 +- .../parquet-converter-statefulset.yaml | 152 ++++++++++++++++++ values.yaml | 51 ++++++ 5 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 templates/parquet-converter/parquet-converter-statefulset.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 425d8530..175d9ef0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased +* [FEATURE] Parquet converter: allow running it as a StatefulSet with a persistent volume per pod #685 * [ENHANCEMENT] Expose `nginx.config.client_body_buffer_size` to set the nginx `client_body_buffer_size` #350 * [DEPENDENCY] Update Helm release rollout-operator to v0.53.1 #683 * [DEPENDENCY] Update prom/memcached-exporter Docker tag to v0.18.0 #684 diff --git a/README.md b/README.md index 458321df..6ca543db 100644 --- a/README.md +++ b/README.md @@ -596,6 +596,13 @@ Kubernetes: `^1.19.0-0` | parquet_converter.​livenessProbe.​httpGet.​path | string | `"/ready"` | | | parquet_converter.​livenessProbe.​httpGet.​port | string | `"http-metrics"` | | | parquet_converter.​nodeSelector | object | `{}` | | +| parquet_converter.​persistentVolume.​accessModes | list | `["ReadWriteOnce"]` | parquet-converter data Persistent Volume access modes Must match those of existing PV or dynamic provisioner Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ | +| parquet_converter.​persistentVolume.​annotations | object | `{}` | parquet-converter data Persistent Volume Claim annotations | +| parquet_converter.​persistentVolume.​enabled | bool | `true` | If true and parquet_converter.statefulSet.enabled is true, the parquet converter will create/use a Persistent Volume Claim If false, use emptyDir | +| parquet_converter.​persistentVolume.​retentionPolicy | object | `{}` | StatefulSetAutoDeletePVC feature https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention | +| parquet_converter.​persistentVolume.​size | string | `"2Gi"` | parquet-converter data Persistent Volume size. The converter downloads a whole block before it converts it, so this must hold the largest block of any tenant. | +| parquet_converter.​persistentVolume.​storageClass | string | `nil` | parquet-converter data Persistent Volume Storage Class If defined, storageClassName: If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. | +| parquet_converter.​persistentVolume.​subPath | string | `""` | Subdirectory of parquet-converter data Persistent Volume to mount Useful if the volume's root directory is not empty | | parquet_converter.​podAnnotations | object | `{"prometheus.io/port":"http-metrics","prometheus.io/scrape":"true"}` | Pod Annotations | | parquet_converter.​podLabels | object | `{}` | Pod Labels | | parquet_converter.​readinessProbe.​httpGet.​path | string | `"/ready"` | | @@ -614,6 +621,9 @@ Kubernetes: `^1.19.0-0` | parquet_converter.​startupProbe.​failureThreshold | int | `10` | | | parquet_converter.​startupProbe.​httpGet.​path | string | `"/ready"` | | | parquet_converter.​startupProbe.​httpGet.​port | string | `"http-metrics"` | | +| parquet_converter.​statefulSet.​enabled | bool | `false` | If true, use a statefulset instead of a deployment for pod management. Each pod then gets its own Persistent Volume Claim (see persistentVolume), which keeps the same volume across restarts and rollouts. This is useful with more than one replica, because the converter downloads each block into its data_dir and replicas must not share one. | +| parquet_converter.​statefulSet.​podManagementPolicy | string | `"Parallel"` | The converter ring does not need ordered starts, so pods start and stop in parallel. ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies | +| parquet_converter.​statefulStrategy.​type | string | `"RollingUpdate"` | | | parquet_converter.​strategy.​rollingUpdate.​maxSurge | int | `0` | | | parquet_converter.​strategy.​rollingUpdate.​maxUnavailable | int | `1` | | | parquet_converter.​strategy.​type | string | `"RollingUpdate"` | | diff --git a/templates/parquet-converter/parquet-converter-dep.yaml b/templates/parquet-converter/parquet-converter-dep.yaml index 461dfc98..b76c8009 100644 --- a/templates/parquet-converter/parquet-converter-dep.yaml +++ b/templates/parquet-converter/parquet-converter-dep.yaml @@ -1,4 +1,4 @@ -{{- if .Values.parquet_converter.enabled }} +{{- if and (not .Values.parquet_converter.statefulSet.enabled) .Values.parquet_converter.enabled }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/templates/parquet-converter/parquet-converter-statefulset.yaml b/templates/parquet-converter/parquet-converter-statefulset.yaml new file mode 100644 index 00000000..39216346 --- /dev/null +++ b/templates/parquet-converter/parquet-converter-statefulset.yaml @@ -0,0 +1,152 @@ +{{- if and .Values.parquet_converter.statefulSet.enabled .Values.parquet_converter.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "cortex.parquetConverterFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cortex.parquetConverterLabels" . | nindent 4 }} + annotations: + {{- toYaml .Values.parquet_converter.annotations | nindent 4 }} +spec: + replicas: {{ .Values.parquet_converter.replicas }} + selector: + matchLabels: + {{- include "cortex.parquetConverterSelectorLabels" . | nindent 6 }} + updateStrategy: + {{- toYaml .Values.parquet_converter.statefulStrategy | nindent 4 }} + podManagementPolicy: "{{ .Values.parquet_converter.statefulSet.podManagementPolicy }}" + serviceName: {{ include "cortex.parquetConverterFullname" . }} + {{- if .Values.parquet_converter.persistentVolume.enabled }} + {{- if semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version }} + {{- with .Values.parquet_converter.persistentVolume.retentionPolicy }} + persistentVolumeClaimRetentionPolicy: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: storage + {{- if .Values.parquet_converter.persistentVolume.annotations }} + annotations: + {{ toYaml .Values.parquet_converter.persistentVolume.annotations | nindent 10 }} + {{- end }} + spec: + {{- if .Values.parquet_converter.persistentVolume.storageClass }} + {{- if (eq "-" .Values.parquet_converter.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.parquet_converter.persistentVolume.storageClass }}" + {{- end }} + {{- end }} + accessModes: + {{- toYaml .Values.parquet_converter.persistentVolume.accessModes | nindent 10 }} + resources: + requests: + storage: "{{ .Values.parquet_converter.persistentVolume.size }}" + {{- end }} + template: + metadata: + labels: + {{- include "cortex.parquetConverterLabels" . | nindent 8 }} + {{- with .Values.parquet_converter.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.parquet_converter.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "cortex.serviceAccountName" . }} + {{- if .Values.parquet_converter.priorityClassName }} + priorityClassName: {{ .Values.parquet_converter.priorityClassName }} + {{- end }} + {{- if .Values.parquet_converter.securityContext.enabled }} + securityContext: {{- omit .Values.parquet_converter.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + initContainers: + {{- toYaml .Values.parquet_converter.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: parquet-converter + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=parquet-converter" + - "-config.file=/etc/cortex/cortex.yaml" + {{- range $key, $value := .Values.parquet_converter.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.parquet_converter.extraVolumeMounts }} + {{- toYaml .Values.parquet_converter.extraVolumeMounts | nindent 12 }} + {{- end }} + - name: config + mountPath: /etc/cortex + - name: runtime-config + mountPath: /etc/cortex-runtime-config + - name: storage + mountPath: "/data" + {{- with .Values.parquet_converter.persistentVolume.subPath }} + subPath: {{ . }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.config.server.http_listen_port }} + protocol: TCP + startupProbe: + {{- toYaml .Values.parquet_converter.startupProbe | nindent 12 }} + livenessProbe: + {{- toYaml .Values.parquet_converter.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.parquet_converter.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.parquet_converter.resources | nindent 12 }} + {{- if .Values.parquet_converter.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.parquet_converter.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.parquet_converter.env }} + env: + {{- toYaml .Values.parquet_converter.env | nindent 12 }} + {{- end }} + {{- with .Values.parquet_converter.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.parquet_converter.extraContainers }} + {{- toYaml .Values.parquet_converter.extraContainers | nindent 8 }} + {{- end }} + nodeSelector: + {{- toYaml .Values.parquet_converter.nodeSelector | nindent 8 }} + {{- if .Values.parquet_converter.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.parquet_converter.topologySpreadConstraints | nindent 8}} + {{- end }} + affinity: + {{- toYaml .Values.parquet_converter.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.parquet_converter.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.parquet_converter.terminationGracePeriodSeconds }} + volumes: + {{- include "cortex.configVolume" . | nindent 8 }} + - name: runtime-config + configMap: + name: {{ template "cortex.fullname" . }}-runtime-config + {{- if not .Values.parquet_converter.persistentVolume.enabled }} + - name: storage + emptyDir: {} + {{- end }} + {{- if .Values.parquet_converter.extraVolumes }} + {{- toYaml .Values.parquet_converter.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/values.yaml b/values.yaml index 5da4d034..ba8b32fc 100644 --- a/values.yaml +++ b/values.yaml @@ -1171,6 +1171,17 @@ parquet_converter: enabled: false replicas: 1 + statefulSet: + # -- If true, use a statefulset instead of a deployment for pod management. + # Each pod then gets its own Persistent Volume Claim (see persistentVolume), + # which keeps the same volume across restarts and rollouts. This is useful + # with more than one replica, because the converter downloads each block + # into its data_dir and replicas must not share one. + enabled: false + # -- The converter ring does not need ordered starts, so pods start and stop in parallel. + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies + podManagementPolicy: Parallel + service: annotations: {} labels: {} @@ -1186,6 +1197,43 @@ parquet_converter: resources: {} + persistentVolume: + # -- If true and parquet_converter.statefulSet.enabled is true, + # the parquet converter will create/use a Persistent Volume Claim + # If false, use emptyDir + enabled: true + + # -- parquet-converter data Persistent Volume Claim annotations + annotations: {} + + # -- parquet-converter data Persistent Volume access modes + # Must match those of existing PV or dynamic provisioner + # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + accessModes: + - ReadWriteOnce + + # -- parquet-converter data Persistent Volume size. + # The converter downloads a whole block before it converts it, so this + # must hold the largest block of any tenant. + size: 2Gi + + # -- Subdirectory of parquet-converter data Persistent Volume to mount + # Useful if the volume's root directory is not empty + subPath: '' + + # -- parquet-converter data Persistent Volume Storage Class + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. + storageClass: null + + # -- StatefulSetAutoDeletePVC feature + # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention + retentionPolicy: {} + # whenDeleted: Retain + # whenScaled: Retain + # -- Additional Cortex container arguments, e.g. log.level (debug, info, warn, error) extraArgs: {} @@ -1239,6 +1287,9 @@ parquet_converter: maxSurge: 0 maxUnavailable: 1 + statefulStrategy: + type: RollingUpdate + terminationGracePeriodSeconds: 180 tolerations: []