diff --git a/pkg/common/utils/k8s/client.go b/pkg/common/utils/k8s/client.go index 60be9a96..8462e568 100644 --- a/pkg/common/utils/k8s/client.go +++ b/pkg/common/utils/k8s/client.go @@ -133,13 +133,12 @@ func ApplyStatefulSet(ctx context.Context, k8sclient client.Client, st *appv1.St return err } } - //check the statefulset need update or not. - ev := equal(st, &est) - // apply pre-processing before create statefulset + // Render the final desired template before comparing, including the BE init image. for _, pasf := range pasfs { pasf(st, nil) } + ev := equal(st, &est) if create { return CreateClientObject(ctx, k8sclient, st) @@ -151,12 +150,47 @@ func ApplyStatefulSet(ctx context.Context, k8sclient client.Client, st *appv1.St return nil } - st.ResourceVersion = est.ResourceVersion - err = PatchClientObject(ctx, k8sclient, st) - if err == nil { - return nil + updated := est.DeepCopy() + // Keep API validation of immutable fields instead of silently accepting unsupported changes. + updated.Spec.ServiceName = st.Spec.ServiceName + updated.Spec.Selector = st.Spec.Selector + updated.Spec.VolumeClaimTemplates = st.Spec.VolumeClaimTemplates + updated.Spec.Replicas = st.Spec.Replicas + updated.Spec.Template = *st.Spec.Template.DeepCopy() + // Preserve metadata supplied by other controllers and rollout tools. + templateMeta := est.Spec.Template.ObjectMeta.DeepCopy() + if templateMeta.Labels == nil { + templateMeta.Labels = map[string]string{} + } + for key, value := range st.Spec.Template.Labels { + templateMeta.Labels[key] = value + } + if templateMeta.Annotations == nil { + templateMeta.Annotations = map[string]string{} + } + for key, value := range st.Spec.Template.Annotations { + templateMeta.Annotations[key] = value + } + updated.Spec.Template.ObjectMeta = *templateMeta + if st.Spec.UpdateStrategy.Type != "" { + updated.Spec.UpdateStrategy = st.Spec.UpdateStrategy + } + if st.Spec.RevisionHistoryLimit != nil { + updated.Spec.RevisionHistoryLimit = st.Spec.RevisionHistoryLimit + } + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + for key, value := range st.Labels { + updated.Labels[key] = value + } + if updated.Annotations == nil { + updated.Annotations = map[string]string{} + } + for key, value := range st.Annotations { + updated.Annotations[key] = value } - return err + return k8sclient.Patch(ctx, updated, client.MergeFromWithOptions(&est, client.MergeFromWithOptimisticLock{})) } func ApplyDorisCluster(ctx context.Context, k8sclient client.Client, dcr *dorisv1.DorisCluster) error { diff --git a/pkg/common/utils/k8s/statefulset_live_state_test.go b/pkg/common/utils/k8s/statefulset_live_state_test.go new file mode 100644 index 00000000..7ab71bb9 --- /dev/null +++ b/pkg/common/utils/k8s/statefulset_live_state_test.go @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package k8s_test + +import ( + "context" + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + v1 "github.com/apache/doris-operator/api/doris/v1" + "github.com/apache/doris-operator/pkg/common/utils/k8s" + "github.com/apache/doris-operator/pkg/common/utils/resource" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "testing" +) + +func liveSTS() *appsv1.StatefulSet { + return &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "fe", Namespace: "test"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(3)), ServiceName: "fe", Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "fe"}}, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{Type: appsv1.RollingUpdateStatefulSetStrategyType, RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{Partition: ptr.To(int32(0))}}, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "fe"}}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "fe", Image: "doris:good", Env: []corev1.EnvVar{{Name: "A", Value: "good"}}}}}}}} +} +func TestApplyStatefulSetRepairsCachedHashDrift(t *testing.T) { + for _, key := range []string{v1.ComponentResourceHash, dv1.DisaggregatedSpecHashValueAnnotation} { + for _, field := range []string{"image", "replicas", "strategy", "env"} { + t.Run(key+"/"+field, func(t *testing.T) { + old := liveSTS() + resource.StatefulsetDeepEqualWithKey(old, old.DeepCopy(), key, false) + switch field { + case "image": + old.Spec.Template.Spec.Containers[0].Image = "wrong" + case "replicas": + old.Spec.Replicas = ptr.To(int32(4)) + case "strategy": + old.Spec.UpdateStrategy.Type = appsv1.OnDeleteStatefulSetStrategyType + old.Spec.UpdateStrategy.RollingUpdate = nil + case "env": + old.Spec.Template.Spec.Containers[0].Env[0].Value = "wrong" + } + c := fake.NewClientBuilder().WithObjects(old).Build() + eq := func(n, o *appsv1.StatefulSet) bool { return resource.StatefulsetDeepEqualWithKey(n, o, key, false) } + if err := k8s.ApplyStatefulSet(context.Background(), c, liveSTS(), eq); err != nil { + t.Fatal(err) + } + got := &appsv1.StatefulSet{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(old), got); err != nil { + t.Fatal(err) + } + if *got.Spec.Replicas != 3 || got.Spec.Template.Spec.Containers[0].Image != "doris:good" || got.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType || got.Spec.Template.Spec.Containers[0].Env[0].Value != "good" { + t.Fatalf("drift remains: %+v", got.Spec) + } + if !eq(liveSTS(), got) { + t.Fatal("not idempotent after repair") + } + }) + } + } +} +func TestLegacyConfigEnvReplicaUpdatePreservesTemplate(t *testing.T) { + old := liveSTS() + old.Spec.Template.Spec.Containers[0].Env = append(old.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "CONFIGMAP_MOUNT_PATH", Value: "/etc/doris"}, corev1.EnvVar{Name: "CONFIGMAP_MOUNT_PATH", Value: "/etc/doris"}) + resource.StatefulsetDeepEqualWithKey(old, old.DeepCopy(), "", false) + old.Spec.Replicas = ptr.To(int32(4)) + c := fake.NewClientBuilder().WithObjects(old).Build() + if err := k8s.ApplyStatefulSet(context.Background(), c, liveSTS(), func(n, o *appsv1.StatefulSet) bool { return resource.StatefulSetDeepEqual(n, o, false) }); err != nil { + t.Fatal(err) + } + got := &appsv1.StatefulSet{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(old), got); err != nil { + t.Fatal(err) + } + if *got.Spec.Replicas != 3 || len(got.Spec.Template.Spec.Containers[0].Env) != 3 { + t.Fatal("legacy environment was changed during replica correction") + } +} + +func TestStatefulSetReconcileWithAPIDefaults(t *testing.T) { + env := &envtest.Environment{} + cfg, err := env.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := env.Stop(); err != nil { + t.Error(err) + } + }) + c, err := client.New(cfg, client.Options{}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := c.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}); err != nil { + t.Fatal(err) + } + desired := liveSTS() + desired.Spec.Template.Spec.Containers[0].Env = append(desired.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}) + desired.Spec.Template.Spec.Containers[0].ReadinessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"true"}}}} + desired.Spec.Template.Spec.Volumes = []corev1.Volume{{Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config"}}}}, {Name: "podinfo", VolumeSource: corev1.VolumeSource{DownwardAPI: &corev1.DownwardAPIVolumeSource{Items: []corev1.DownwardAPIVolumeFile{{Path: "name", FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}}}}} + resource.StatefulSetDeepEqual(desired, desired.DeepCopy(), false) + created := desired.DeepCopy() + if err := c.Create(ctx, created); err != nil { + t.Fatal(err) + } + if !resource.StatefulSetDeepEqual(desired.DeepCopy(), created, false) { + t.Fatalf("API defaults produce a difference: %+v", created.Spec.Template.Spec) + } + created.Spec.Template.Annotations = map[string]string{"external.example/rollout": "keep"} + created.Spec.Template.Spec.Containers[0].Image = "doris:wrong" + created.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType} + if err := c.Update(ctx, created); err != nil { + t.Fatal(err) + } + equal := func(n, o *appsv1.StatefulSet) bool { return resource.StatefulSetDeepEqual(n, o, false) } + if err := k8s.ApplyStatefulSet(ctx, c, desired.DeepCopy(), equal); err != nil { + t.Fatal(err) + } + got := &appsv1.StatefulSet{} + if err := c.Get(ctx, client.ObjectKeyFromObject(created), got); err != nil { + t.Fatal(err) + } + if got.Spec.Template.Spec.Containers[0].Image != "doris:good" || got.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType || got.Spec.Template.Annotations["external.example/rollout"] != "keep" { + t.Fatal("managed drift or external metadata incorrect after repair") + } + rv := got.ResourceVersion + if err := k8s.ApplyStatefulSet(ctx, c, desired.DeepCopy(), equal); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(created), got); err != nil { + t.Fatal(err) + } + if got.ResourceVersion != rv { + t.Fatal("repeated reconcile patched unchanged workload") + } +} + +func TestStatefulSetApplyReturnsImmutableFieldError(t *testing.T) { + env := &envtest.Environment{} + cfg, err := env.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := env.Stop(); err != nil { + t.Error(err) + } + }) + c, err := client.New(cfg, client.Options{}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := c.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}); err != nil { + t.Fatal(err) + } + current := liveSTS() + if err := c.Create(ctx, current); err != nil { + t.Fatal(err) + } + desired := liveSTS() + desired.Spec.ServiceName = "different-headless-service" + err = k8s.ApplyStatefulSet(ctx, c, desired, func(n, o *appsv1.StatefulSet) bool { return resource.StatefulSetDeepEqual(n, o, false) }) + if !apierrors.IsInvalid(err) { + t.Fatalf("immutable change was silently ignored: %v", err) + } +} diff --git a/pkg/common/utils/resource/statefulset.go b/pkg/common/utils/resource/statefulset.go index 41063899..df5e6721 100644 --- a/pkg/common/utils/resource/statefulset.go +++ b/pkg/common/utils/resource/statefulset.go @@ -23,8 +23,9 @@ import ( "github.com/apache/doris-operator/pkg/common/utils/metadata" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" + "k8s.io/utils/ptr" ) const ( @@ -90,133 +91,115 @@ func NewStatefulSet(dcr *v1.DorisCluster, config map[string]interface{}, compone return st } -// StatefulSetDeepEqual judge two statefulset equal or not. -func StatefulSetDeepEqual(new *appv1.StatefulSet, old *appv1.StatefulSet, excludeReplicas bool) bool { - equal := StatefulsetDeepEqualWithKey(new, old, v1.ComponentResourceHash, excludeReplicas) - if !equal { - return clear_config_env_path_numbers_alwaysEquals(new, old, v1.ComponentResourceHash, excludeReplicas) - +// StatefulSetDeepEqual compares live managed fields and preserves harmless +// duplicate CONFIGMAP_MOUNT_PATH entries left by older native operators. +func StatefulSetDeepEqual(new, old *appv1.StatefulSet, excludeReplicas bool) bool { + if StatefulsetDeepEqualWithKey(new, old, v1.ComponentResourceHash, excludeReplicas) { + return true } - - return true + return clear_config_env_path_numbers_alwaysEquals(new, old, v1.ComponentResourceHash, excludeReplicas) } -// clear duplicate env for compatible 1.4.1, 1.5.1,1.6.1 -// Note, Note,Note, please delete the logic code in 2024.12.31, -// used deepcopy for envs will assign to new and the operation will result new updated in unexpected. -func clear_config_env_path_numbers_alwaysEquals(new *appv1.StatefulSet, old *appv1.StatefulSet, annoKey string, excludeReplicas bool) bool { - newAvoidEnvsSt := new.DeepCopy() - var newEnvs []corev1.EnvVar - //for find the main container position for assign envs to new equal. - nMainContainerIndex := -1 - for i, c := range new.Spec.Template.Spec.Containers { - if c.Name == string(v1.Component_FE) || c.Name == string(v1.Component_BE) || c.Name == string(v1.Component_CN) || c.Name == string(v1.Component_Broker) { - for _, env := range c.Env { - newEnv := env.DeepCopy() - newEnvs = append(newEnvs, *newEnv) - } - nMainContainerIndex = i - break +func clear_config_env_path_numbers_alwaysEquals(new, old *appv1.StatefulSet, annoKey string, excludeReplicas bool) bool { + candidate := new.DeepCopy() + for i := range candidate.Spec.Template.Spec.Containers { + container := &candidate.Spec.Template.Spec.Containers[i] + if container.Name != string(v1.Component_FE) && container.Name != string(v1.Component_BE) && container.Name != string(v1.Component_CN) && container.Name != string(v1.Component_Broker) { + continue } - } - - var oldEnvs []corev1.EnvVar - for _, c := range old.Spec.Template.Spec.Containers { - if c.Name == string(v1.Component_FE) || c.Name == string(v1.Component_BE) || c.Name == string(v1.Component_CN) || c.Name == string(v1.Component_Broker) { - oldEnvs = c.Env - break + var previous *corev1.Container + for j := range old.Spec.Template.Spec.Containers { + if old.Spec.Template.Spec.Containers[j].Name == container.Name { + previous = &old.Spec.Template.Spec.Containers[j] + break + } } - } - - for i := 0; i < len(newEnvs); i++ { - if newEnvs[i].Name == config_env_name { - newEnvs = append(newEnvs[:i], newEnvs[i+1:]...) - i-- + if previous == nil { + return false } - } - - poss := []int{} - for i := 0; i < len(oldEnvs); i++ { - if oldEnvs[i].Name == config_env_name { - poss = append(poss, i) + var expected []corev1.EnvVar + for _, env := range container.Env { + if env.Name == config_env_name { + if env.Value != config_env_path || env.ValueFrom != nil { + return false + } + } else { + expected = append(expected, env) + } } - } - for i := 0; i < len(poss); i++ { - if poss[i] >= len(newEnvs) { - newEnvs = append(newEnvs, corev1.EnvVar{Name: config_env_name, Value: config_env_path}) - } else { - index := poss[i] - lastEnvs := append([]corev1.EnvVar{{Name: config_env_name, Value: config_env_path}}, newEnvs[index:]...) - newEnvs = append(newEnvs[:index], lastEnvs...) + previousCount, desiredCount := 0, 0 + for _, env := range previous.Env { + if env.Name == config_env_name { + previousCount++ + } } - } - - if len(newEnvs) != len(oldEnvs) { - return false - } else { - for i := 0; i < len(newEnvs); i++ { - if newEnvs[i].Name != oldEnvs[i].Name { - return false + for _, env := range container.Env { + if env.Name == config_env_name { + desiredCount++ } - if newEnvs[i].Value != "" && newEnvs[i].Value != oldEnvs[i].Value { - return false + } + // This exception is only for surplus duplicates from older versions. + // Never suppress adding a newly requested configuration environment. + if previousCount < 2 || previousCount <= desiredCount { + return false + } + var compatible []corev1.EnvVar + next := 0 + for _, env := range previous.Env { + if env.Name == config_env_name { + if env.Value != config_env_path || env.ValueFrom != nil { + return false + } + compatible = append(compatible, env) + } else { + if next >= len(expected) { + return false + } + compatible = append(compatible, expected[next]) + next++ } } - } - - newAvoidEnvsSt.Spec.Template.Spec.Containers[nMainContainerIndex].Env = newEnvs - if *new.Spec.Replicas == *old.Spec.Replicas { - nAvoidOb := statefulSetHashObject(newAvoidEnvsSt, excludeReplicas) - nAvoidHash := hash.HashObject(nAvoidOb) - oldHash := old.Annotations[annoKey] - return nAvoidHash == oldHash - } else { - *newAvoidEnvsSt.Spec.Replicas = *old.Spec.Replicas - nAvoidOb := statefulSetHashObject(newAvoidEnvsSt, excludeReplicas) - nAvoidHash := hash.HashObject(nAvoidOb) - oldHash := old.Annotations[annoKey] - if nAvoidHash == oldHash { - new.Spec.Template.Spec.Containers[nMainContainerIndex].Env = newEnvs + if next != len(expected) { + return false } - return false + container.Env = compatible + // Only preserve the legacy layout when every other managed template field + // matches. Replica changes must not trigger a rolling update of these pods. + if !StatefulsetDeepEqualWithKey(candidate, old, annoKey, true) { + return false + } + new.Spec.Template.Spec.Containers[i].Env = compatible + return StatefulsetDeepEqualWithKey(new, old, annoKey, excludeReplicas) } + return false } func StatefulsetDeepEqualWithKey(new, old *appv1.StatefulSet, annoKey string, excludeReplicas bool) bool { - /* if omit { - newHso := statefulSetHashObject(new, excludeReplicas) - newHashv := hash.HashObject(newHso) - oldHso := statefulSetHashObject(old, excludeReplicas) - oldHashv := hash.HashObject(oldHso) - return new.Namespace == old.Namespace && newHashv == oldHashv - }*/ - var newHashv, oldHashv string if annoKey == "" { annoKey = v1.ComponentResourceHash } - - if _, ok := new.Annotations[annoKey]; ok { - newHashv = new.Annotations[annoKey] - } else { - newHso := statefulSetHashObject(new, excludeReplicas) - newHashv = hash.HashObject(newHso) + annotations := NewAnnotations(Annotations(new.Annotations)) + annotations[annoKey] = hash.HashObject(statefulSetHashObject(new, excludeReplicas)) + new.Annotations = annotations + if new.Name != old.Name || new.Namespace != old.Namespace { + return false } - - if _, ok := old.Annotations[annoKey]; ok { - oldHashv = old.Annotations[annoKey] - } else { - oldHso := statefulSetHashObject(old, excludeReplicas) - oldHashv = hash.HashObject(oldHso) + if !excludeReplicas && ptr.Deref(new.Spec.Replicas, int32(1)) != ptr.Deref(old.Spec.Replicas, int32(1)) { + return false } - - anno := Annotations{} - anno.AddAnnotation(new.Annotations) - anno.Add(annoKey, newHashv) - new.Annotations = anno - - klog.Info("the statefulset name "+new.Name+" new hash value ", newHashv, " old have value ", oldHashv) - return newHashv == oldHashv && - new.Namespace == old.Namespace + if new.Spec.ServiceName != old.Spec.ServiceName || !equality.Semantic.DeepEqual(new.Spec.Selector, old.Spec.Selector) { + return false + } + for key, value := range new.Labels { + if actual, ok := old.Labels[key]; !ok || actual != value { + return false + } + } + if !statefulSetStrategyEqual(new.Spec.UpdateStrategy, old.Spec.UpdateStrategy) { + return false + } + return equality.Semantic.DeepEqual(normalizedPodTemplate(new.Spec.Template, new.Spec.Template), normalizedPodTemplate(old.Spec.Template, new.Spec.Template)) && + equality.Semantic.DeepEqual(normalizedClaimTemplates(new.Spec.VolumeClaimTemplates), normalizedClaimTemplates(old.Spec.VolumeClaimTemplates)) } // hashStatefulsetObject contains the info for hash comparison. diff --git a/pkg/common/utils/resource/statefulset_normalize.go b/pkg/common/utils/resource/statefulset_normalize.go new file mode 100644 index 00000000..4224078a --- /dev/null +++ b/pkg/common/utils/resource/statefulset_normalize.go @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package resource + +import ( + "encoding/json" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "strings" +) + +func statefulSetStrategyEqual(desired, actual appsv1.StatefulSetUpdateStrategy) bool { + // The CG business comparison deliberately clears strategy; its graceful + // state machine compares and manages this field separately. + if desired.Type == "" { + return true + } + if actual.Type == "" { + actual.Type = appsv1.RollingUpdateStatefulSetStrategyType + } + if desired.Type != actual.Type { + return false + } + if desired.Type == appsv1.OnDeleteStatefulSetStrategyType { + return true + } + partition := func(s appsv1.StatefulSetUpdateStrategy) int32 { + if s.RollingUpdate == nil { + return 0 + } + return ptr.Deref(s.RollingUpdate.Partition, int32(0)) + } + if partition(desired) != partition(actual) { + return false + } + if desired.RollingUpdate != nil && desired.RollingUpdate.MaxUnavailable != nil { + return actual.RollingUpdate != nil && equality.Semantic.DeepEqual(desired.RollingUpdate.MaxUnavailable, actual.RollingUpdate.MaxUnavailable) + } + return true +} +func normalizedPodTemplate(template, desired corev1.PodTemplateSpec) corev1.PodTemplateSpec { + // JSON round-tripping removes empty slices/maps as the API does. + data, _ := json.Marshal(template) + var result corev1.PodTemplateSpec + _ = json.Unmarshal(data, &result) + labels, annotations := map[string]string{}, map[string]string{} + for key := range desired.Labels { + if value, ok := result.Labels[key]; ok { + labels[key] = value + } + } + for key := range desired.Annotations { + if value, ok := result.Annotations[key]; ok { + annotations[key] = value + } + } + result.ObjectMeta = metav1.ObjectMeta{Labels: labels, Annotations: annotations} + pod := &result.Spec + if pod.RestartPolicy == "" { + pod.RestartPolicy = corev1.RestartPolicyAlways + } + if pod.DNSPolicy == "" { + pod.DNSPolicy = corev1.DNSClusterFirst + } + if pod.SchedulerName == "" { + pod.SchedulerName = corev1.DefaultSchedulerName + } + if pod.TerminationGracePeriodSeconds == nil { + pod.TerminationGracePeriodSeconds = ptr.To(int64(30)) + } + if pod.SecurityContext == nil { + pod.SecurityContext = &corev1.PodSecurityContext{} + } + if pod.EnableServiceLinks == nil { + pod.EnableServiceLinks = ptr.To(true) + } + if pod.PreemptionPolicy == nil { + pod.PreemptionPolicy = ptr.To(corev1.PreemptLowerPriority) + } + normalizeContainers(pod.Containers) + normalizeContainers(pod.InitContainers) + for i := range pod.Volumes { + volume := &pod.Volumes[i] + if volume.Secret != nil && volume.Secret.DefaultMode == nil { + volume.Secret.DefaultMode = ptr.To(int32(420)) + } + if volume.ConfigMap != nil && volume.ConfigMap.DefaultMode == nil { + volume.ConfigMap.DefaultMode = ptr.To(int32(420)) + } + if volume.DownwardAPI != nil { + if volume.DownwardAPI.DefaultMode == nil { + volume.DownwardAPI.DefaultMode = ptr.To(int32(420)) + } + for j := range volume.DownwardAPI.Items { + if ref := volume.DownwardAPI.Items[j].FieldRef; ref != nil && ref.APIVersion == "" { + ref.APIVersion = "v1" + } + } + } + if volume.Projected != nil { + if volume.Projected.DefaultMode == nil { + volume.Projected.DefaultMode = ptr.To(int32(420)) + } + for j := range volume.Projected.Sources { + if source := volume.Projected.Sources[j].DownwardAPI; source != nil { + for k := range source.Items { + if ref := source.Items[k].FieldRef; ref != nil && ref.APIVersion == "" { + ref.APIVersion = "v1" + } + } + } + } + } + if volume.HostPath != nil && volume.HostPath.Type == nil { + volume.HostPath.Type = ptr.To(corev1.HostPathUnset) + } + } + return result +} +func normalizeContainers(containers []corev1.Container) { + for i := range containers { + c := &containers[i] + if c.ImagePullPolicy == "" { + c.ImagePullPolicy = corev1.PullIfNotPresent + image := c.Image + if !strings.Contains(image, "@") { + last := image[strings.LastIndex(image, "/")+1:] + if !strings.Contains(last, ":") || strings.HasSuffix(last, ":latest") { + c.ImagePullPolicy = corev1.PullAlways + } + } + } + if c.TerminationMessagePath == "" { + c.TerminationMessagePath = corev1.TerminationMessagePathDefault + } + if c.TerminationMessagePolicy == "" { + c.TerminationMessagePolicy = corev1.TerminationMessageReadFile + } + for j := range c.Ports { + if c.Ports[j].Protocol == "" { + c.Ports[j].Protocol = corev1.ProtocolTCP + } + } + for j := range c.Env { + if c.Env[j].ValueFrom != nil { + if ref := c.Env[j].ValueFrom.FieldRef; ref != nil && ref.APIVersion == "" { + ref.APIVersion = "v1" + } + } + } + for _, probe := range []*corev1.Probe{c.LivenessProbe, c.ReadinessProbe, c.StartupProbe} { + if probe == nil { + continue + } + if probe.TimeoutSeconds == 0 { + probe.TimeoutSeconds = 1 + } + if probe.PeriodSeconds == 0 { + probe.PeriodSeconds = 10 + } + if probe.SuccessThreshold == 0 { + probe.SuccessThreshold = 1 + } + if probe.FailureThreshold == 0 { + probe.FailureThreshold = 3 + } + if probe.HTTPGet != nil && probe.HTTPGet.Scheme == "" { + probe.HTTPGet.Scheme = corev1.URISchemeHTTP + } + } + if c.Lifecycle != nil { + for _, handler := range []*corev1.LifecycleHandler{c.Lifecycle.PostStart, c.Lifecycle.PreStop} { + if handler != nil && handler.HTTPGet != nil && handler.HTTPGet.Scheme == "" { + handler.HTTPGet.Scheme = corev1.URISchemeHTTP + } + } + } + } +} +func normalizedClaimTemplates(claims []corev1.PersistentVolumeClaim) []corev1.PersistentVolumeClaim { + if len(claims) == 0 { + return nil + } + result := make([]corev1.PersistentVolumeClaim, len(claims)) + for i := range claims { + result[i] = *claims[i].DeepCopy() + result[i].TypeMeta = metav1.TypeMeta{} + result[i].ObjectMeta = metav1.ObjectMeta{Name: claims[i].Name, Labels: claims[i].Labels, Annotations: claims[i].Annotations} + result[i].Status = corev1.PersistentVolumeClaimStatus{} + if result[i].Spec.VolumeMode == nil { + result[i].Spec.VolumeMode = ptr.To(corev1.PersistentVolumeFilesystem) + } + } + return result +} diff --git a/pkg/common/utils/resource/statefulset_test.go b/pkg/common/utils/resource/statefulset_test.go index f81417c0..30f3bdd1 100644 --- a/pkg/common/utils/resource/statefulset_test.go +++ b/pkg/common/utils/resource/statefulset_test.go @@ -80,7 +80,8 @@ func Test_StatefulsetSetDeepEqual(t *testing.T) { nsts := []*appv1.StatefulSet{nst, envNotEqualNst, annoExistEqualNst} osts := []*appv1.StatefulSet{ost, envNotEqualOst, annoExistEqualOst} - ress := []bool{true, false, true} + // Harmless legacy duplicates are compared against live fields, without requiring a hash annotation. + ress := []bool{true, true, true} for i := 0; i < len(nsts); i++ { res := StatefulSetDeepEqual(nsts[i], osts[i], false) if res != ress[i] { @@ -88,3 +89,11 @@ func Test_StatefulsetSetDeepEqual(t *testing.T) { } } } +func TestLegacyCompatibilityDoesNotHideNewConfigEnv(t *testing.T) { + old := &appv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "fe", Namespace: "test"}, Spec: appv1.StatefulSetSpec{Replicas: GetInt32Pointer(1), Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "fe", Image: "fe:test", Env: []corev1.EnvVar{{Name: "A", Value: "value"}}}}}}}} + desired := old.DeepCopy() + desired.Spec.Template.Spec.Containers[0].Env = append(desired.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: config_env_name, Value: config_env_path}) + if StatefulSetDeepEqual(desired, old, false) { + t.Fatal("legacy compatibility hid a newly configured environment variable") + } +} diff --git a/pkg/controller/sub_controller/cn/controller.go b/pkg/controller/sub_controller/cn/controller.go index eeb1c256..613a0bef 100644 --- a/pkg/controller/sub_controller/cn/controller.go +++ b/pkg/controller/sub_controller/cn/controller.go @@ -179,10 +179,9 @@ func (cn *Controller) applyStatefulSet(ctx context.Context, st *appv1.StatefulSe return err } //if the spec is changed, update the status of cn on src. - var excludeReplica bool - //if replicas =0 and not the first time, exclude the hash for autoscaler - if st.Spec.Replicas == nil && !autoscaler { - excludeReplica = true + excludeReplica := autoscaler || st.Spec.Replicas == nil + if excludeReplica { + st.Spec.Replicas = est.Spec.Replicas } //the statefulset equal should exclude pvc. pvc not allowed update when use statefulset manage, when use `operator` mode for management that pvc not allow updated in statetfulset spec. diff --git a/pkg/controller/sub_controller/cn/statefulset_live_state_test.go b/pkg/controller/sub_controller/cn/statefulset_live_state_test.go new file mode 100644 index 00000000..2f93dc99 --- /dev/null +++ b/pkg/controller/sub_controller/cn/statefulset_live_state_test.go @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package cn + +import ( + "context" + "github.com/apache/doris-operator/pkg/common/utils/resource" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "testing" +) + +func TestHPAOwnedCNTemplateUpdatePreservesActualReplicas(t *testing.T) { + for _, initial := range []*int32{nil, ptr.To(int32(3))} { + t.Run(func() string { + if initial == nil { + return "omitted initial replicas" + } + return "explicit initial replicas" + }(), func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Errorf("HPA update panicked: %v", r) + } + }() + old := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "cn", Namespace: "test"}, Spec: appsv1.StatefulSetSpec{Replicas: initial, Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "cn", Image: "old"}}}}}} + resource.StatefulsetDeepEqualWithKey(old, old.DeepCopy(), "", false) + old.Spec.Replicas = ptr.To(int32(5)) + c := newCnController(t, old) + desired := old.DeepCopy() + desired.ResourceVersion = "" + desired.Annotations = nil + desired.Spec.Replicas = initial + desired.Spec.Template.Spec.Containers[0].Image = "new" + if err := c.applyStatefulSet(context.Background(), desired, true); err != nil { + t.Fatal(err) + } + got := &appsv1.StatefulSet{} + if err := c.K8sclient.Get(context.Background(), client.ObjectKeyFromObject(old), got); err != nil { + t.Fatal(err) + } + if *got.Spec.Replicas != 5 || got.Spec.Template.Spec.Containers[0].Image != "new" { + t.Fatalf("replicas=%d image=%s", *got.Spec.Replicas, got.Spec.Template.Spec.Containers[0].Image) + } + }) + } +} diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go index ebddfd73..5967b84f 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go @@ -226,6 +226,7 @@ func (dcgs *DisaggregatedComputeGroupsController) reconcileInternalService(ctx c // reconcileStatefulset return bool means reconcile print error message. func (dcgs *DisaggregatedComputeGroupsController) reconcileStatefulset(ctx context.Context, st *appv1.StatefulSet, cluster *dv1.DorisDisaggregatedCluster, cg *dv1.ComputeGroup) (*sc.Event, error) { + dcgs.useNewDefaultValuesInStatefulset(st) //use new default value before apply new statefulset, when creating and apply spec change. ndf := func(st *appv1.StatefulSet, est *appv1.StatefulSet) { dcgs.useNewDefaultValuesInStatefulset(st) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset_live_state_test.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset_live_state_test.go new file mode 100644 index 00000000..321f8dfb --- /dev/null +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset_live_state_test.go @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package computegroups + +import ( + "context" + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + "github.com/apache/doris-operator/pkg/common/utils/resource" + sc "github.com/apache/doris-operator/pkg/controller/sub_controller" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestReconcileStatefulsetCorrectsActualReplicaDrift(t *testing.T) { + desired := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "component", Namespace: "audit"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(3)), Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "good"}}}}}} + old := desired.DeepCopy() + resource.StatefulsetDeepEqualWithKey(old, old.DeepCopy(), dv1.DisaggregatedSpecHashValueAnnotation, false) + old.Spec.Replicas = ptr.To(int32(2)) + c := fake.NewClientBuilder().WithObjects(old).Build() + controller := &DisaggregatedComputeGroupsController{DisaggregatedSubDefaultController: sc.DisaggregatedSubDefaultController{K8sclient: c}} + ddc := &dv1.DorisDisaggregatedCluster{Spec: dv1.DorisDisaggregatedClusterSpec{FeSpec: dv1.FeSpec{CommonSpec: dv1.CommonSpec{Replicas: ptr.To(int32(3))}}}} + ddc.Status.ComputeGroupStatuses = []dv1.ComputeGroupStatus{{UniqueId: "cg", Phase: dv1.Ready}} + if _, err := controller.reconcileStatefulset(context.Background(), desired, ddc, &dv1.ComputeGroup{UniqueId: "cg", CommonSpec: dv1.CommonSpec{Replicas: ptr.To(int32(3))}}); err != nil { + t.Fatal(err) + } + got := &appsv1.StatefulSet{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(old), got); err != nil { + t.Fatal(err) + } + if *got.Spec.Replicas != 3 { + t.Fatalf("replica drift remains: %d", *got.Spec.Replicas) + } +} diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset_live_state_test.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset_live_state_test.go new file mode 100644 index 00000000..b56a5b8c --- /dev/null +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset_live_state_test.go @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package disaggregated_fe + +import ( + "context" + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + "github.com/apache/doris-operator/pkg/common/utils/resource" + sc "github.com/apache/doris-operator/pkg/controller/sub_controller" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestReconcileStatefulsetCorrectsActualReplicaDrift(t *testing.T) { + desired := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "component", Namespace: "audit"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(3)), Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "good"}}}}}} + old := desired.DeepCopy() + resource.StatefulsetDeepEqualWithKey(old, old.DeepCopy(), dv1.DisaggregatedSpecHashValueAnnotation, false) + old.Spec.Replicas = ptr.To(int32(2)) + c := fake.NewClientBuilder().WithObjects(old).Build() + controller := &DisaggregatedFEController{DisaggregatedSubDefaultController: sc.DisaggregatedSubDefaultController{K8sclient: c}} + ddc := &dv1.DorisDisaggregatedCluster{Spec: dv1.DorisDisaggregatedClusterSpec{FeSpec: dv1.FeSpec{CommonSpec: dv1.CommonSpec{Replicas: ptr.To(int32(3))}}}} + if _, err := controller.reconcileStatefulset(context.Background(), desired, ddc); err != nil { + t.Fatal(err) + } + got := &appsv1.StatefulSet{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(old), got); err != nil { + t.Fatal(err) + } + if *got.Spec.Replicas != 3 { + t.Fatalf("replica drift remains: %d", *got.Spec.Replicas) + } +} diff --git a/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset_live_state_test.go b/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset_live_state_test.go new file mode 100644 index 00000000..a80616c1 --- /dev/null +++ b/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset_live_state_test.go @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package metaservice + +import ( + "context" + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + "github.com/apache/doris-operator/pkg/common/utils/resource" + sc "github.com/apache/doris-operator/pkg/controller/sub_controller" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestReconcileStatefulsetCorrectsActualReplicaDrift(t *testing.T) { + desired := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "component", Namespace: "audit"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(3)), Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "good"}}}}}} + old := desired.DeepCopy() + resource.StatefulsetDeepEqualWithKey(old, old.DeepCopy(), dv1.DisaggregatedSpecHashValueAnnotation, false) + old.Spec.Replicas = ptr.To(int32(2)) + c := fake.NewClientBuilder().WithObjects(old).Build() + controller := &DisaggregatedMSController{DisaggregatedSubDefaultController: sc.DisaggregatedSubDefaultController{K8sclient: c}} + ddc := &dv1.DorisDisaggregatedCluster{Spec: dv1.DorisDisaggregatedClusterSpec{FeSpec: dv1.FeSpec{CommonSpec: dv1.CommonSpec{Replicas: ptr.To(int32(3))}}}} + if _, err := controller.reconcileStatefulset(context.Background(), desired, ddc); err != nil { + t.Fatal(err) + } + got := &appsv1.StatefulSet{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(old), got); err != nil { + t.Fatal(err) + } + if *got.Spec.Replicas != 3 { + t.Fatalf("replica drift remains: %d", *got.Spec.Replicas) + } +}