diff --git a/doc/operation/fe_heap.md b/doc/operation/fe_heap.md new file mode 100644 index 00000000..9e7bdf1a --- /dev/null +++ b/doc/operation/fe_heap.md @@ -0,0 +1,72 @@ +# FE heap defaults in containers + +For newly created DorisCluster and DorisDisaggregatedCluster resources, the +operator adapts image-provided FE JVM defaults when `limits.memory` is set: + +| Configuration entry | Heap defaults | +| --- | --- | +| `JAVA_OPTS` (JDK 8) | `-Xms` and `-Xmx` both set to 80% of the limit, rounded down to MiB | +| `JAVA_OPTS_FOR_JDK_17` | `-XX:InitialRAMPercentage=80 -XX:MaxRAMPercentage=80` | + +The fixed image `-Xms` and `-Xmx` values are removed before adding these defaults. +Other image JVM options, including GC, logging and module access, are retained. +The JDK 17 calculation requires the image JVM to recognize the container memory +limit correctly. Verify the effective heap on your supported image/cgroup +combination. The 80% heap budget leaves roughly 20% of the limit for other +container memory usage. + +No adaptation is performed without a memory limit. `requests.memory` is not a +heap budget. This feature does not change BE JVM settings or Helm templates. + +## User configuration + +Each explicit JVM entry in a user `fe.conf` takes precedence as a whole. For +example, setting `JAVA_OPTS_FOR_JDK_17` preserves that exact value, whether it +uses fixed or percentage heap sizes. The operator does not merge defaults into +that entry or override it even if it equals an old image default. An omitted +entry is populated from the corresponding image default, if the image has it. +ConfigMaps and image files are never edited. Other files in the configuration +mount are copied along with `fe.conf`. + +The standard `/etc/doris` configuration directory (or a literal +`CONFIGMAP_MOUNT_PATH`) is prepared using a same-image init container and an +`emptyDir`. Direct mounts over the FE home/conf directory and custom +`DORIS_HOME`/`DORIS_ROOT` environments remain user-managed. Explicit JVM +environment entries are also left alone; whether the image startup script +honors them is unchanged. Prefer `fe.conf` for overrides. + +The preparation runs before optional runtime configuration persistence. It does +not require that feature or a PVC, and preserves its configuration changes. + +## Existing clusters and operator upgrades + +Before creating workloads or initializing status, the operator saves a decision +in the CR annotation `doris.apache.org/fe-heap-policy`: + +- `auto`: adapt default heaps when a memory limit is configured. +- `legacy`: retain the previous behavior and Pod template for this feature. + +A CR with existing status or an owned StatefulSet defaults to `legacy`, including +StatefulSets with older non-controller owner references. New CRs default to +`auto`. The annotation survives operator restarts, scaling and StatefulSet +recreation. Do not remove it to change policies. + +To opt an existing cluster in during a planned FE update: + +```sh +kubectl annotate dcr -n doris.apache.org/fe-heap-policy=auto --overwrite +# For disaggregated clusters, use ddc instead of dcr. +``` + +This changes the desired FE Pod template and uses the existing update workflow. +Explicit user JVM entries still take precedence. Set the annotation to `legacy` +on a new CR to opt out. Changing an already active cluster back to `legacy` also +changes its template and restores image defaults for entries not supplied by the +user; plan this as an FE configuration update. + +For verification, inspect `fe.conf` inside the running FE and the JVM effective +flags (`jcmd VM.flags` where available). For an 8Gi limit, JDK 8 should use +`-Xms6553m -Xmx6553m`; JDK 17 should have initial and maximum heaps around 6.4Gi, +subject to JVM alignment/ergonomics. Include a real startup check on both JDKs, +custom fixed/percentage settings, and an upgrade check that existing FE Pod +UIDs and templates remain unchanged until explicitly enabled. diff --git a/pkg/common/utils/feheap/feheap_test.go b/pkg/common/utils/feheap/feheap_test.go new file mode 100644 index 00000000..b2d1b181 --- /dev/null +++ b/pkg/common/utils/feheap/feheap_test.go @@ -0,0 +1,393 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feheap + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + v1 "github.com/apache/doris-operator/api/doris/v1" + "github.com/apache/doris-operator/pkg/common/utils/runtimeconfig" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func testClient(objects ...client.Object) client.Client { + scheme := runtime.NewScheme() + _ = v1.AddToScheme(scheme) + _ = dv1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() +} + +func testOwner(ddc bool) client.Object { + meta := metav1.ObjectMeta{Name: "cluster", Namespace: "test", UID: "cluster-uid"} + if ddc { + return &dv1.DorisDisaggregatedCluster{ObjectMeta: meta} + } + return &v1.DorisCluster{ObjectMeta: meta} +} + +func TestPolicySurvivesWorkloadRecreation(t *testing.T) { + for _, ddc := range []bool{false, true} { + for _, scenario := range []string{"new", "status", "legacy-owner", "foreign-owner"} { + t.Run(map[bool]string{false: "dcr/", true: "ddc/"}[ddc]+scenario, func(t *testing.T) { + owner := testOwner(ddc) + st := testWorkload("8Gi") + objects := []client.Object{owner} + want := Auto + switch scenario { + case "status": + switch o := owner.(type) { + case *v1.DorisCluster: + o.Status.FEStatus = &v1.ComponentStatus{} + case *dv1.DorisDisaggregatedCluster: + o.Status.FEStatus = dv1.FEStatus{ClusterId: "existing"} + } + want = Legacy + case "legacy-owner", "foreign-owner": + uid := owner.GetUID() + if scenario == "foreign-owner" { + uid = "other" + } else { + want = Legacy + } + st.OwnerReferences = []metav1.OwnerReference{{UID: uid}} + objects = append(objects, st) + } + c := testClient(objects...) + if err := EnsurePolicy(context.Background(), c, owner); err != nil { + t.Fatal(err) + } + if owner.GetAnnotations()[PolicyAnnotation] != want { + t.Fatalf("annotations: %v", owner.GetAnnotations()) + } + if scenario == "legacy-owner" { + if err := c.Delete(context.Background(), st); err != nil { + t.Fatal(err) + } + } + // Read back the persisted decision, as after an operator restart. + reloaded := testOwner(ddc) + if err := c.Get(context.Background(), client.ObjectKeyFromObject(owner), reloaded); err != nil { + t.Fatal(err) + } + if err := EnsurePolicy(context.Background(), c, reloaded); err != nil { + t.Fatal(err) + } + if reloaded.GetAnnotations()[PolicyAnnotation] != want { + t.Fatal("decision changed") + } + }) + } + } +} + +func TestExplicitPolicy(t *testing.T) { + for _, policy := range []string{Auto, Legacy, "invalid"} { + owner := testOwner(false) + owner.SetAnnotations(map[string]string{PolicyAnnotation: policy}) + err := EnsurePolicy(context.Background(), testClient(owner), owner) + if (err != nil) != (policy == "invalid") { + t.Fatalf("policy %s: %v", policy, err) + } + } +} + +func testWorkload(limit string) *appsv1.StatefulSet { + main := corev1.Container{Name: "fe", Image: "doris:test", ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/opt/apache-doris/fe_entrypoint.sh"}, Args: []string{"$(FE_ADDR)"}, + Env: []corev1.EnvVar{{Name: "DORIS_ROOT", Value: "/opt/apache-doris"}}, + VolumeMounts: []corev1.VolumeMount{{Name: "config", MountPath: "/etc/doris", ReadOnly: true}}} + if limit != "" { + main.Resources.Limits = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse(limit)} + } + return &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "cluster-fe", Namespace: "test"}, Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{main}, Volumes: []corev1.Volume{{Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "user-config"}}}}}, + }}}} +} + +func TestPrepareBoundaries(t *testing.T) { + for _, scenario := range []string{"legacy", "unset-policy", "no-limit", "request-only", "direct-conf", "custom-root", "env-override"} { + t.Run(scenario, func(t *testing.T) { + owner := testOwner(false) + owner.SetAnnotations(map[string]string{PolicyAnnotation: Auto}) + st := testWorkload("8Gi") + switch scenario { + case "legacy": + owner.SetAnnotations(map[string]string{PolicyAnnotation: Legacy}) + case "unset-policy": + owner.SetAnnotations(nil) + case "no-limit", "request-only": + st.Spec.Template.Spec.Containers[0].Resources.Limits = nil + if scenario == "request-only" { + st.Spec.Template.Spec.Containers[0].Resources.Requests = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("8Gi")} + } + case "direct-conf": + st.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath = feHome + "/conf" + case "custom-root": + st.Spec.Template.Spec.Containers[0].Env[0].Value = "/custom" + case "env-override": + st.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "JAVA_OPTS", Value: "custom"}, {Name: "JAVA_OPTS_FOR_JDK_17", Value: "custom17"}} + } + before := st.DeepCopy() + if err := Prepare(owner, st); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, st) { + t.Fatal("unchanged path mutated workload") + } + }) + } +} + +func TestPrepareLimitsAndIdempotentRendering(t *testing.T) { + for _, limit := range []string{"4Gi", "8Gi", "16Gi", "5000Mi"} { + t.Run(limit, func(t *testing.T) { + owner := testOwner(false) + owner.SetAnnotations(map[string]string{PolicyAnnotation: Auto}) + st := testWorkload(limit) + before := st.DeepCopy() + if err := Prepare(owner, st); err != nil { + t.Fatal(err) + } + pod := st.Spec.Template.Spec + wantHeap := map[string]string{"4Gi": "3276", "8Gi": "6553", "16Gi": "13107", "5000Mi": "4000"}[limit] + foundHeap := false + for _, env := range pod.InitContainers[0].Env { + if env.Name == "FE_HEAP_JDK8_MIB" { + foundHeap = env.Value == wantHeap + } + } + if !foundHeap { + t.Fatalf("wrong JDK8 heap for %s", limit) + } + if len(pod.InitContainers) != 1 || pod.InitContainers[0].Image != pod.Containers[0].Image { + t.Fatal("missing same-image initializer") + } + if !reflect.DeepEqual(pod.Containers[0].Command, before.Spec.Template.Spec.Containers[0].Command) || !reflect.DeepEqual(pod.Containers[0].Args, before.Spec.Template.Spec.Containers[0].Args) { + t.Fatal("entrypoint changed") + } + if pod.Volumes[0].ConfigMap.Name != "user-config" || pod.InitContainers[0].VolumeMounts[1].Name != "config" { + t.Fatal("source ConfigMap changed") + } + if err := Prepare(owner, before); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, st) { + t.Fatal("re-render changed workload") + } + }) + } +} + +func TestComposesWithRuntimePersistence(t *testing.T) { + owner := testOwner(false) + owner.SetAnnotations(map[string]string{PolicyAnnotation: Auto, runtimeconfig.PolicyAnnotation: runtimeconfig.Auto}) + st := testWorkload("8Gi") + st.Spec.Template.Spec.Containers[0].VolumeMounts = append(st.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "log", MountPath: feHome + "/log"}) + st.Spec.Template.Spec.Volumes = append(st.Spec.Template.Spec.Volumes, corev1.Volume{Name: "log", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "logs"}}}) + if err := Prepare(owner, st); err != nil { + t.Fatal(err) + } + if err := runtimeconfig.Prepare(context.Background(), testClient(), nil, owner, st, nil); err != nil { + t.Fatal(err) + } + pod := st.Spec.Template.Spec + if len(pod.InitContainers) != 2 || pod.InitContainers[0].Name != initName { + t.Fatal("wrong preparation order") + } + found := false + for _, mount := range pod.InitContainers[1].VolumeMounts { + if mount.MountPath == "/runtime-source" && mount.Name == configVolume { + found = true + } + } + if !found { + t.Fatal("runtime preparation does not consume heap output") + } + found = false + for _, mount := range pod.Containers[0].VolumeMounts { + if mount.MountPath == runtimeconfig.RuntimeDir { + found = true + } + } + if !found { + t.Fatal("runtime persistence mount lost") + } + + root := t.TempDir() + home, source, heapDir := filepath.Join(root, "fe"), filepath.Join(root, "source"), filepath.Join(root, "heap") + store, prepared := filepath.Join(root, "store"), filepath.Join(root, "runtime") + for _, dir := range []string{home + "/conf", source, heapDir, store, prepared} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(home+"/conf/fe.conf", []byte(imageConfig), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source+"/fe_custom.conf", []byte("test_override=true"), 0644); err != nil { + t.Fatal(err) + } + replace := strings.NewReplacer("/runtime-source", heapDir, "/runtime-prepared", prepared, "/runtime-store", store) + for i, init := range pod.InitContainers { + cmd := exec.Command("bash", "-ec", replace.Replace(init.Command[2])) + cmd.Env = os.Environ() + for _, env := range init.Env { + value := env.Value + switch env.Name { + case "FE_HEAP_HOME", "RUNTIME_HOME": + value = home + case "FE_HEAP_SOURCE": + value = source + case "FE_HEAP_OUTPUT": + value = heapDir + case "POD_NAME": + value = "cluster-fe-0" + } + cmd.Env = append(cmd.Env, env.Name+"="+value) + } + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("initializer %d: %s: %v", i, output, err) + } + } + got, err := os.ReadFile(prepared + "/fe.conf") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(got), "MaxRAMPercentage=80") || !strings.Contains(string(got), "custom_config_dir = /var/lib/doris/runtime-config") { + t.Fatalf("composed config: %s", got) + } + override, err := os.ReadFile(store + "/.doris-runtime-config/cluster-fe-0/fe_custom.conf") + if err != nil || string(override) != "test_override=true" { + t.Fatalf("runtime override lost: %s: %v", override, err) + } +} + +const imageConfig = `# image configuration +JAVA_OPTS="-Dfile.encoding=UTF-8 -Xms8192m -Xmx8192m -XX:+UseG1GC -Xloggc:$LOG_DIR/fe.gc.log.$CUR_DATE" +JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError --add-opens=java.base/java.lang=ALL-UNNAMED -Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time" +http_port=8030 +` + +func runPrepareScript(t *testing.T, defaults, source, heap string, env ...string) string { + t.Helper() + root := t.TempDir() + home := filepath.Join(root, "fe") + src := filepath.Join(root, "source") + out := filepath.Join(root, "prepared") + for _, dir := range []string{filepath.Join(home, "conf"), src, out} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(home, "conf", "fe.conf"), []byte(defaults), 0644); err != nil { + t.Fatal(err) + } + if source != "" { + if err := os.WriteFile(filepath.Join(src, "fe.conf"), []byte(source), 0644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(src, "log4j2.xml"), []byte("sibling"), 0644); err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-ec", prepareScript) + cmd.Env = append(os.Environ(), "FE_HEAP_HOME="+home, "FE_HEAP_SOURCE="+src, "FE_HEAP_OUTPUT="+out, "FE_HEAP_JDK8_MIB="+heap, "FE_HEAP_PRESERVE_JDK8=false", "FE_HEAP_PRESERVE_JDK17=false") + cmd.Env = append(cmd.Env, env...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("script: %s: %v", output, err) + } + raw, err := os.ReadFile(filepath.Join(out, "fe.conf")) + if err != nil { + t.Fatal(err) + } + original, _ := os.ReadFile(filepath.Join(home, "conf", "fe.conf")) + if string(original) != defaults { + t.Fatal("image source changed") + } + if source != "" { + original, _ = os.ReadFile(filepath.Join(src, "fe.conf")) + if string(original) != source { + t.Fatal("user source changed") + } + } + sibling, _ := os.ReadFile(filepath.Join(out, "log4j2.xml")) + if string(sibling) != "sibling" { + t.Fatal("sibling file lost") + } + return string(raw) +} + +func TestScriptAdaptsImageDefaults(t *testing.T) { + for _, heap := range []string{"3276", "6553", "13107"} { + got := runPrepareScript(t, imageConfig, "", heap) + for _, want := range []string{"-Xms" + heap + "m -Xmx" + heap + "m", "-XX:InitialRAMPercentage=80 -XX:MaxRAMPercentage=80", "--add-opens=java.base/java.lang=ALL-UNNAMED", "$LOG_DIR/fe.gc.log.$CUR_DATE", "http_port=8030"} { + if !strings.Contains(got, want) { + t.Fatalf("missing %q in %s", want, got) + } + } + if strings.Count(got, "-Xms") != 1 || strings.Count(got, "-Xmx") != 1 { + t.Fatalf("unexpected extra fixed heap flags: %s", got) + } + } +} + +func TestScriptPreservesUserJVMSettings(t *testing.T) { + for _, source := range []string{ + "JAVA_OPTS_FOR_JDK_17=\"-Xms8192m -Xmx8192m -Dcustom=true\"\nhttp_port=18030\n", + "JAVA_OPTS_FOR_JDK_17='-XX:InitialRAMPercentage=10 -XX:MaxRAMPercentage=60'\n", + "JAVA_OPTS=\"-Xmx512m\"\nJAVA_OPTS_FOR_JDK_17=\"-Xmx1g\"\n", + "JAVA_OPTS_FOR_JDK_17=\n", + } { + got := runPrepareScript(t, imageConfig, source, "3276") + for _, line := range strings.Split(strings.TrimSpace(source), "\n") { + if !strings.Contains(got, line) { + t.Fatalf("user setting lost: %s in %s", line, got) + } + } + if strings.Contains(got, "MaxRAMPercentage=80") { + t.Fatal("overwrote user JDK17 settings") + } + } +} + +func TestScriptFillsMissingJVMSettings(t *testing.T) { + got := runPrepareScript(t, imageConfig, "# JAVA_OPTS=ignored\nhttp_port=18030\ncustom_config_dir=/custom\n", "3276") + if !strings.Contains(got, "-Xms3276m -Xmx3276m") || !strings.Contains(got, "MaxRAMPercentage=80") || !strings.Contains(got, "custom_config_dir=/custom") || !strings.Contains(got, "http_port=18030") { + t.Fatal(got) + } + // JDK 8 images need not contain JDK 17 options; do not create an incomplete entry. + got = runPrepareScript(t, "JAVA_OPTS='-Xmx8192m -XX:+UseG1GC'\n", "", "3276") + if strings.Contains(got, "JAVA_OPTS_FOR_JDK_17") || !strings.Contains(got, "-Xms3276m -Xmx3276m") { + t.Fatal(got) + } +} diff --git a/pkg/common/utils/feheap/pod.go b/pkg/common/utils/feheap/pod.go new file mode 100644 index 00000000..c067b528 --- /dev/null +++ b/pkg/common/utils/feheap/pod.go @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feheap + +import ( + _ "embed" + "fmt" + "path" + "strconv" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + initName = "prepare-fe-heap" + configVolume = "fe-heap-prepared" + feHome = "/opt/apache-doris/fe" +) + +//go:embed prepare.sh +var prepareScript string + +// Prepare stages heap defaults before optional runtime configuration preparation. +// The latter consumes this directory and retains its own custom_config_dir changes. +// Heap adaptation also works independently, without persistent storage. +func Prepare(owner client.Object, st *appsv1.StatefulSet) error { + if owner.GetAnnotations()[PolicyAnnotation] != Auto { + return nil + } + pod := &st.Spec.Template.Spec + for i := range pod.Containers { + main := &pod.Containers[i] + if main.Name != "fe" { + continue + } + limit := main.Resources.Limits.Memory() + if limit.Sign() <= 0 { + return nil + } + // Dividing by 1.25 MiB gives 80% of the limit in whole MiB. + const bytesPerMiB = 1 << 20 + heapMiB := limit.Value() / (bytesPerMiB * 5 / 4) + if heapMiB < 1 { + return fmt.Errorf("FE memory limit is too small to allocate a heap") + } + preserve8, preserve17 := false, false + source := "/etc/doris" + for _, env := range main.Env { + switch env.Name { + case "JAVA_OPTS": + preserve8 = true + case "JAVA_OPTS_FOR_JDK_17": + preserve17 = true + case "CONFIGMAP_MOUNT_PATH": + // Dynamic/custom config locations are user-managed. + if env.ValueFrom != nil { + return nil + } + if env.Value != "" { + source = env.Value + } + case "DORIS_ROOT": + if env.ValueFrom != nil || (env.Value != "" && path.Clean(env.Value) != path.Dir(feHome)) { + return nil + } + case "DORIS_HOME": + if env.ValueFrom != nil || (env.Value != "" && path.Clean(env.Value) != feHome) { + return nil + } + } + } + if preserve8 && preserve17 { + return nil + } + for _, mount := range main.VolumeMounts { + // Never rewrite directly mounted user-managed FE configuration. + if mount.MountPath == "/opt/apache-doris" || mount.MountPath == feHome || + mount.MountPath == feHome+"/conf" || strings.HasPrefix(mount.MountPath, feHome+"/conf/") { + return nil + } + } + for _, c := range pod.InitContainers { + if c.Name == initName { + return fmt.Errorf("init container name %s is reserved", initName) + } + } + for _, v := range pod.Volumes { + if v.Name == configVolume { + return fmt.Errorf("volume name %s is reserved", configVolume) + } + } + init := corev1.Container{ + Name: initName, Image: main.Image, ImagePullPolicy: main.ImagePullPolicy, + Command: []string{"/bin/bash", "-ec", prepareScript}, + Env: []corev1.EnvVar{ + {Name: "FE_HEAP_HOME", Value: feHome}, + {Name: "FE_HEAP_SOURCE", Value: "/heap-source"}, + {Name: "FE_HEAP_OUTPUT", Value: "/heap-prepared"}, + {Name: "FE_HEAP_JDK8_MIB", Value: strconv.FormatInt(heapMiB, 10)}, + {Name: "FE_HEAP_PRESERVE_JDK8", Value: strconv.FormatBool(preserve8)}, + {Name: "FE_HEAP_PRESERVE_JDK17", Value: strconv.FormatBool(preserve17)}, + }, + Resources: main.Resources, SecurityContext: main.SecurityContext, + VolumeMounts: []corev1.VolumeMount{{Name: configVolume, MountPath: "/heap-prepared"}}, + } + found := false + for j := range main.VolumeMounts { + mount := &main.VolumeMounts[j] + if path.Clean(mount.MountPath) != path.Clean(source) { + continue + } + original := *mount + original.MountPath = "/heap-source" + original.ReadOnly = true + init.VolumeMounts = append(init.VolumeMounts, original) + mount.Name = configVolume + mount.ReadOnly = false + mount.SubPath = "" + mount.SubPathExpr = "" + found = true + } + if !found { + main.VolumeMounts = append(main.VolumeMounts, corev1.VolumeMount{Name: configVolume, MountPath: source}) + } + pod.Volumes = append(pod.Volumes, corev1.Volume{Name: configVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) + pod.InitContainers = append(pod.InitContainers, init) + return nil + } + return nil +} diff --git a/pkg/common/utils/feheap/policy.go b/pkg/common/utils/feheap/policy.go new file mode 100644 index 00000000..be01bfda --- /dev/null +++ b/pkg/common/utils/feheap/policy.go @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package feheap adapts image-provided FE heap defaults to container memory limits. +package feheap + +import ( + "context" + "fmt" + "reflect" + + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + v1 "github.com/apache/doris-operator/api/doris/v1" + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + PolicyAnnotation = "doris.apache.org/fe-heap-policy" + Auto = "auto" + Legacy = "legacy" +) + +// EnsurePolicy saves the decision before initializing status or creating workloads. +// An old cluster must not silently opt in when its StatefulSet is later recreated. +func EnsurePolicy(ctx context.Context, c client.Client, owner client.Object) error { + if !owner.GetDeletionTimestamp().IsZero() { + return nil + } + switch policy := owner.GetAnnotations()[PolicyAnnotation]; policy { + case Auto, Legacy: + return nil + case "": + default: + return fmt.Errorf("invalid %s value %q: expected auto or legacy", PolicyAnnotation, policy) + } + var hasStatus bool + switch o := owner.(type) { + case *v1.DorisCluster: + hasStatus = !reflect.ValueOf(o.Status).IsZero() + case *dv1.DorisDisaggregatedCluster: + hasStatus = !reflect.ValueOf(o.Status).IsZero() + default: + return fmt.Errorf("unsupported FE heap owner %T", owner) + } + policy := Auto + if hasStatus { + policy = Legacy + } else { + var workloads appsv1.StatefulSetList + if err := c.List(ctx, &workloads, client.InNamespace(owner.GetNamespace())); err != nil { + return err + } + for _, st := range workloads.Items { + for _, ref := range st.OwnerReferences { + // Older operators did not set Controller=true. + if owner.GetUID() != "" && ref.UID == owner.GetUID() { + policy = Legacy + } + } + } + } + before := owner.DeepCopyObject().(client.Object) + annotations := owner.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[PolicyAnnotation] = policy + owner.SetAnnotations(annotations) + return c.Patch(ctx, owner, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) +} diff --git a/pkg/common/utils/feheap/prepare.sh b/pkg/common/utils/feheap/prepare.sh new file mode 100644 index 00000000..8662d8d9 --- /dev/null +++ b/pkg/common/utils/feheap/prepare.sh @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail +if [[ -d "$FE_HEAP_SOURCE" ]]; then + cp -RL "$FE_HEAP_SOURCE/." "$FE_HEAP_OUTPUT/" +fi +image_conf="$FE_HEAP_HOME/conf/fe.conf" +conf="$FE_HEAP_OUTPUT/fe.conf" +if [[ ! -f "$conf" ]]; then + cp "$image_conf" "$conf" +fi + +# Read only image defaults, without evaluating their shell expressions. User +# JAVA_OPTS entries are preserved as whole settings, including percentage heaps. +adapt_heap() { + local key="$1" heap="$2" preserve="$3" replacement + if [[ "$preserve" == true ]] || { [[ -f "$FE_HEAP_SOURCE/fe.conf" ]] && + grep -Eq "^[[:space:]]*${key}[[:space:]]*=" "$FE_HEAP_SOURCE/fe.conf"; }; then + return + fi + replacement=$(awk -v key="$key" -v heap="$heap" ' + $0 ~ "^[[:space:]]*" key "[[:space:]]*=" { + value = $0 + sub(/^[^=]*=[[:space:]]*/, "", value) + sub(/[[:space:]]*$/, "", value) + quote = substr(value, 1, 1) + if (quote == "\"" || quote == "\047") { + value = substr(value, 2, length(value) - 2) + } else { + quote = "\"" + } + gsub(/(^|[[:space:]])-Xm[sx][^[:space:]]*/, "", value) + gsub(/(^|[[:space:]])-XX:(InitialRAMPercentage|MaxRAMPercentage|InitialHeapSize|MaxHeapSize)=[^[:space:]]*/, "", value) + print key "=" quote value " " heap quote + exit + } + ' "$image_conf") + # Do not manufacture JVM options for JDKs unsupported by this image. + if [[ -z "$replacement" ]]; then + return + fi + awk -v key="$key" '$0 !~ "^[[:space:]]*" key "[[:space:]]*=" { print }' "$conf" > "$conf.tmp" + printf '\n%s\n' "$replacement" >> "$conf.tmp" + mv "$conf.tmp" "$conf" +} + +adapt_heap JAVA_OPTS "-Xms${FE_HEAP_JDK8_MIB}m -Xmx${FE_HEAP_JDK8_MIB}m" "$FE_HEAP_PRESERVE_JDK8" +adapt_heap JAVA_OPTS_FOR_JDK_17 '-XX:InitialRAMPercentage=80 -XX:MaxRAMPercentage=80' "$FE_HEAP_PRESERVE_JDK17" diff --git a/pkg/controller/disaggregated_cluster_controller.go b/pkg/controller/disaggregated_cluster_controller.go index 8ea172ab..6532768b 100644 --- a/pkg/controller/disaggregated_cluster_controller.go +++ b/pkg/controller/disaggregated_cluster_controller.go @@ -25,6 +25,7 @@ import ( "time" dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + "github.com/apache/doris-operator/pkg/common/utils/feheap" "github.com/apache/doris-operator/pkg/common/utils/hash" "github.com/apache/doris-operator/pkg/common/utils/runtimeconfig" sc "github.com/apache/doris-operator/pkg/controller/sub_controller" @@ -224,6 +225,10 @@ func (dc *DisaggregatedClusterReconciler) Reconcile(ctx context.Context, req rec if err != nil { return ctrl.Result{}, err } + if err := feheap.EnsurePolicy(ctx, dc.Client, &ddc); err != nil { + return ctrl.Result{}, err + } + if err := runtimeconfig.EnsurePolicy(ctx, dc.Client, &ddc); err != nil { dc.Recorder.Event(&ddc, corev1.EventTypeWarning, "RuntimeConfigPersistenceBlocked", err.Error()) return ctrl.Result{}, err diff --git a/pkg/controller/doriscluster_controller.go b/pkg/controller/doriscluster_controller.go index 27ecdff6..55d6660d 100644 --- a/pkg/controller/doriscluster_controller.go +++ b/pkg/controller/doriscluster_controller.go @@ -36,6 +36,7 @@ package controller import ( "context" dorisv1 "github.com/apache/doris-operator/api/doris/v1" + "github.com/apache/doris-operator/pkg/common/utils/feheap" "github.com/apache/doris-operator/pkg/common/utils/k8s" "github.com/apache/doris-operator/pkg/common/utils/resource" "github.com/apache/doris-operator/pkg/common/utils/runtimeconfig" @@ -136,6 +137,10 @@ func (r *DorisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } + if err := feheap.EnsurePolicy(ctx, r.Client, dcr); err != nil { + return ctrl.Result{}, err + } + if err := runtimeconfig.EnsurePolicy(ctx, r.Client, dcr); err != nil { r.Recorder.Event(dcr, corev1.EventTypeWarning, "RuntimeConfigPersistenceBlocked", err.Error()) return ctrl.Result{}, err diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go index d85d9ab9..8e0ea847 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "github.com/apache/doris-operator/pkg/common/utils/feheap" "github.com/apache/doris-operator/pkg/common/utils/runtimeconfig" "strconv" "strings" @@ -89,6 +90,9 @@ func (dfc *DisaggregatedFEController) Sync(ctx context.Context, obj client.Objec svc := dfc.newService(ddc, confMap) st := dfc.NewStatefulset(ddc, confMap) + if err := feheap.Prepare(ddc, st); err != nil { + return err + } if err := runtimeconfig.Prepare(ctx, dfc.K8sclient, dfc.K8srecorder, ddc, st, confMap); err != nil { return err } diff --git a/pkg/controller/sub_controller/fe/controller.go b/pkg/controller/sub_controller/fe/controller.go index cebc800e..13831d65 100644 --- a/pkg/controller/sub_controller/fe/controller.go +++ b/pkg/controller/sub_controller/fe/controller.go @@ -20,6 +20,7 @@ package fe import ( "context" v1 "github.com/apache/doris-operator/api/doris/v1" + "github.com/apache/doris-operator/pkg/common/utils/feheap" "github.com/apache/doris-operator/pkg/common/utils/k8s" "github.com/apache/doris-operator/pkg/common/utils/resource" "github.com/apache/doris-operator/pkg/common/utils/runtimeconfig" @@ -131,6 +132,9 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error } st := fc.buildFEStatefulSet(cluster, config) + if err := feheap.Prepare(cluster, &st); err != nil { + return err + } if err := runtimeconfig.Prepare(ctx, fc.K8sclient, fc.K8srecorder, cluster, &st, config); err != nil { return err }