diff --git a/config/operator/disaggregated-operator.yaml b/config/operator/disaggregated-operator.yaml index 6641442d..bb1f8ce5 100644 --- a/config/operator/disaggregated-operator.yaml +++ b/config/operator/disaggregated-operator.yaml @@ -179,6 +179,7 @@ rules: resources: - secrets verbs: + - create - get - update - list diff --git a/config/operator/operator-sign-ns.yaml b/config/operator/operator-sign-ns.yaml index fa2005fd..68c0a849 100644 --- a/config/operator/operator-sign-ns.yaml +++ b/config/operator/operator-sign-ns.yaml @@ -178,6 +178,7 @@ rules: resources: - secrets verbs: + - create - get - update - list diff --git a/config/operator/operator.yaml b/config/operator/operator.yaml index 6adf837e..de630c01 100644 --- a/config/operator/operator.yaml +++ b/config/operator/operator.yaml @@ -171,6 +171,7 @@ rules: resources: - secrets verbs: + - create - get - update - list diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d5016a6c..5475eae1 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -91,8 +91,10 @@ rules: resources: - secrets verbs: + - create - get - list + - update - watch - apiGroups: - "" diff --git a/doc/examples/disaggregated/cluster/ddc-pwd.yaml b/doc/examples/disaggregated/cluster/ddc-pwd.yaml index 050720da..713b1fe6 100644 --- a/doc/examples/disaggregated/cluster/ddc-pwd.yaml +++ b/doc/examples/disaggregated/cluster/ddc-pwd.yaml @@ -39,4 +39,15 @@ spec: image: apache/doris:be-3.0.3 - uniqueId: cg2 replicas: 3 - image: apache/doris:be-3.0.3 \ No newline at end of file + image: apache/doris:be-3.0.3 + +# New clusters receive the adminUser environment variables from an owned Secret. +# On upgrade, existing clusters keep their current credential delivery. To migrate +# during a maintenance window, apply the updated Secret RBAC and verify that +# the operator has its usual compute-group rollout permissions in the managed +# namespace (create pods/exec and delete pods, as in disaggregated-operator.yaml). +# Annotate the DDC and wait for FE and compute-group rollouts: +# kubectl annotate ddc -n doris.apache.org/admin-credential-mode=secret --overwrite +# The adminUser fields remain in the DDC. Use spec.authSecret alone to keep +# credentials outside the DDC. If both are configured, the environment still +# uses adminUser and the existing authSecret mount is preserved. diff --git a/helm-charts/doris-operator/templates/clusterrole.yaml b/helm-charts/doris-operator/templates/clusterrole.yaml index fe9b18db..384e0650 100644 --- a/helm-charts/doris-operator/templates/clusterrole.yaml +++ b/helm-charts/doris-operator/templates/clusterrole.yaml @@ -92,6 +92,7 @@ rules: resources: - secrets verbs: + - create - get - update - list diff --git a/pkg/common/utils/mysql/mysql.go b/pkg/common/utils/mysql/mysql.go index 4dd89b9a..cd345c4a 100644 --- a/pkg/common/utils/mysql/mysql.go +++ b/pkg/common/utils/mysql/mysql.go @@ -77,7 +77,10 @@ func NewDorisSqlDB(cfg DBConfig, tlsConfig *TLSConfig, secret *corev1.Secret) (* } rootCertPool := x509.NewCertPool() - if tlsConfig != nil && secret != nil { + if tlsConfig != nil { + if secret == nil { + return nil, errors.New("management TLS requires a certificate Secret") + } ca := secret.Data[tlsConfig.CAFileName] clientCert := secret.Data[tlsConfig.ClientCertFileName] clientKey := secret.Data[tlsConfig.ClientKeyFileName] diff --git a/pkg/controller/disaggregated_cluster_controller.go b/pkg/controller/disaggregated_cluster_controller.go index 345eade0..f2f2a1c2 100644 --- a/pkg/controller/disaggregated_cluster_controller.go +++ b/pkg/controller/disaggregated_cluster_controller.go @@ -206,6 +206,7 @@ func fdbClusterFile(obj client.Object) (string, bool) { func (dc *DisaggregatedClusterReconciler) resourceBuilder(builder *ctrl.Builder) *ctrl.Builder { return builder.For(&dv1.DorisDisaggregatedCluster{}). + Owns(&corev1.Secret{}). Owns(&appv1.StatefulSet{}, controller_builder.MatchEveryOwner). Owns(&corev1.Service{}, controller_builder.MatchEveryOwner) } @@ -234,6 +235,11 @@ func (dc *DisaggregatedClusterReconciler) Reconcile(ctx context.Context, req rec return ctrl.Result{}, err } + if err := sc.EnsureAdminCredentials(ctx, dc.Client, &ddc); err != nil { + dc.Recorder.Event(&ddc, corev1.EventTypeWarning, "AdminCredentialsPending", err.Error()) + return ctrl.Result{}, err + } + hv := hash.HashObject(ddc.Spec) var res ctrl.Result diff --git a/pkg/controller/doriscluster_controller.go b/pkg/controller/doriscluster_controller.go index 622f37b3..153b3d14 100644 --- a/pkg/controller/doriscluster_controller.go +++ b/pkg/controller/doriscluster_controller.go @@ -96,7 +96,7 @@ var ( //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get //+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch +//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete diff --git a/pkg/controller/sub_controller/admin_credentials.go b/pkg/controller/sub_controller/admin_credentials.go new file mode 100644 index 00000000..e3c844d0 --- /dev/null +++ b/pkg/controller/sub_controller/admin_credentials.go @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sub_controller + +import ( + "context" + "fmt" + "reflect" + + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + "github.com/apache/doris-operator/pkg/common/utils/resource" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + adminCredentialMode = "doris.apache.org/admin-credential-mode" + adminCredentialRevision = "doris.apache.org/admin-credential-revision" + modeSecret = "secret" + modeLegacy = "legacy" +) + +func adminSecretName(ddc *dv1.DorisDisaggregatedCluster) string { + return "doris-admin-" + string(ddc.UID) +} + +// EnsureAdminCredentials establishes the cluster-wide mode before any child is rendered. +func EnsureAdminCredentials(ctx context.Context, c client.Client, ddc *dv1.DorisDisaggregatedCluster) error { + if !ddc.DeletionTimestamp.IsZero() { + return nil + } + mode := ddc.Annotations[adminCredentialMode] + if mode == "" { + existing, err := hasExistingDeployment(ctx, c, ddc) + if err != nil { + return err + } + mode = modeSecret + if existing { + mode = modeLegacy + } + } + if mode != modeSecret && mode != modeLegacy { + return fmt.Errorf("unknown %s value %q", adminCredentialMode, mode) + } + revision := "" + if mode == modeSecret && ddc.Spec.AdminUser != nil { + secret, err := reconcileAdminSecret(ctx, c, ddc) + if err != nil { + return err + } + revision = secret.Annotations[adminCredentialRevision] + } + if mode == ddc.Annotations[adminCredentialMode] && revision == ddc.Annotations[adminCredentialRevision] { + return nil + } + before := ddc.DeepCopy() + if ddc.Annotations == nil { + ddc.Annotations = make(map[string]string) + } + ddc.Annotations[adminCredentialMode] = mode + if revision == "" { + delete(ddc.Annotations, adminCredentialRevision) + } else { + ddc.Annotations[adminCredentialRevision] = revision + } + return c.Patch(ctx, ddc, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) +} + +func hasExistingDeployment(ctx context.Context, c client.Client, ddc *dv1.DorisDisaggregatedCluster) (bool, error) { + if !reflect.ValueOf(ddc.Status).IsZero() { + return true, nil + } + var workloads appsv1.StatefulSetList + if err := c.List(ctx, &workloads, client.InNamespace(ddc.Namespace)); err != nil { + return false, err + } + for _, st := range workloads.Items { + // Older releases created non-controller owner references. + for _, owner := range st.OwnerReferences { + if ddc.UID != "" && owner.UID == ddc.UID { + return true, nil + } + } + } + return false, nil +} + +func reconcileAdminSecret(ctx context.Context, c client.Client, ddc *dv1.DorisDisaggregatedCluster) (*corev1.Secret, error) { + name := adminSecretName(ddc) + data := map[string][]byte{corev1.BasicAuthUsernameKey: []byte(ddc.Spec.AdminUser.Name), corev1.BasicAuthPasswordKey: []byte(ddc.Spec.AdminUser.Password)} + secret := &corev1.Secret{} + err := c.Get(ctx, client.ObjectKey{Namespace: ddc.Namespace, Name: name}, secret) + if apierrors.IsNotFound(err) { + owner := metav1.NewControllerRef(ddc, dv1.GroupVersion.WithKind("DorisDisaggregatedCluster")) + owner.BlockOwnerDeletion = nil + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: ddc.Namespace, Name: name, + OwnerReferences: []metav1.OwnerReference{*owner}, + Annotations: map[string]string{adminCredentialRevision: string(uuid.NewUUID())}, + }, + Data: data, + } + return secret, c.Create(ctx, secret) + } + if err != nil { + return nil, err + } + if !metav1.IsControlledBy(secret, ddc) { + return nil, fmt.Errorf("Secret %s/%s already exists with a different owner", ddc.Namespace, name) + } + if secret.DeletionTimestamp != nil { + return nil, fmt.Errorf("waiting for deletion of Secret %s/%s", ddc.Namespace, name) + } + // A metadata-only edit must not restart workloads that consume this Secret. + if reflect.DeepEqual(secret.Data, data) && secret.Annotations[adminCredentialRevision] != "" { + return secret, nil + } + secret.Data = data + if secret.Annotations == nil { + secret.Annotations = make(map[string]string) + } + secret.Annotations[adminCredentialRevision] = string(uuid.NewUUID()) + return secret, c.Update(ctx, secret) +} + +func AdminCredentialEnvs(ddc *dv1.DorisDisaggregatedCluster) []corev1.EnvVar { + user := ddc.Spec.AdminUser + if user == nil { + return nil + } + if ddc.Annotations[adminCredentialMode] != modeSecret { + return []corev1.EnvVar{{Name: resource.ADMIN_USER, Value: user.Name}, {Name: resource.ADMIN_PASSWD, Value: user.Password}} + } + ref := func(key string) *corev1.EnvVarSource { + return &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: adminSecretName(ddc)}, Key: key}} + } + return []corev1.EnvVar{{Name: resource.ADMIN_USER, ValueFrom: ref(corev1.BasicAuthUsernameKey)}, {Name: resource.ADMIN_PASSWD, ValueFrom: ref(corev1.BasicAuthPasswordKey)}} +} + +// Secret-backed environment variables are read once when the container starts. +func SetAdminCredentialRevision(ddc *dv1.DorisDisaggregatedCluster, pts *corev1.PodTemplateSpec) { + if rev := ddc.Annotations[adminCredentialRevision]; rev != "" && ddc.Annotations[adminCredentialMode] == modeSecret && ddc.Spec.AdminUser != nil { + if pts.Annotations == nil { + pts.Annotations = make(map[string]string) + } + pts.Annotations[adminCredentialRevision] = rev + } +} diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go index 5967b84f..ac4ce39e 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/controller.go @@ -824,12 +824,19 @@ func (dcgs *DisaggregatedComputeGroupsController) UpdateComponentStatus(obj clie func (dcgs *DisaggregatedComputeGroupsController) recordComputeGroupIds(ddc *dv1.DorisDisaggregatedCluster) error { // get user and password - adminUserName, password := dcgs.GetManagementAdminUserAndPWD(context.Background(), ddc) + ctx := context.Background() + adminUserName, password, err := dcgs.GetManagementAdminUserAndPWD(ctx, ddc) + if err != nil { + return err + } // get host and port // When the operator and dcr are deployed in different namespace, it will be inaccessible, so need to add the dcr svc namespace host := ddc.GetFEVIPAddresss() - confMap := dcgs.GetConfigValuesFromConfigMaps(ddc.Namespace, resource.FE_RESOLVEKEY, ddc.Spec.FeSpec.ConfigMaps) + confMap, err := dcgs.ManagementFEConfig(ctx, ddc) + if err != nil { + return err + } queryPort := resource.GetPort(confMap, resource.QUERY_PORT) cfg := mysql.NewDBConfig() cfg.User = adminUserName @@ -837,8 +844,10 @@ func (dcgs *DisaggregatedComputeGroupsController) recordComputeGroupIds(ddc *dv1 cfg.Host = host cfg.Port = strconv.FormatInt(int64(queryPort), 10) - tlsConfig, secretName := dcgs.DisaggregatedSubDefaultController.FindSecretTLSConfig(confMap, ddc) - secret, _ := k8s.GetSecret(context.Background(), dcgs.K8sclient, ddc.Namespace, secretName) + tlsConfig, secret, err := sc.LoadManagementTLS(ctx, dcgs.K8sclient, ddc.Namespace, confMap, ddc.Spec.FeSpec.Secrets) + if err != nil { + return err + } db, err := mysql.NewDorisSqlDB(cfg, tlsConfig, secret) if err != nil { diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/decommission.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/decommission.go index a57d91a9..f9c8310c 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/decommission.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/decommission.go @@ -27,7 +27,6 @@ import ( "time" dv1 "github.com/apache/doris-operator/api/disaggregated/v1" - "github.com/apache/doris-operator/pkg/common/utils/k8s" ms "github.com/apache/doris-operator/pkg/common/utils/metaservice" "github.com/apache/doris-operator/pkg/common/utils/mysql" "github.com/apache/doris-operator/pkg/common/utils/resource" @@ -307,7 +306,10 @@ func (dcgs *DisaggregatedComputeGroupsController) decommissionSnapshot(ctx conte if len(frontends) == 0 { return nil, errors.New("no FE observations are available") } - feConf := dcgs.GetConfigValuesFromConfigMaps(cluster.Namespace, resource.FE_RESOLVEKEY, cluster.Spec.FeSpec.ConfigMaps) + feConf, err := dcgs.ManagementFEConfig(ctx, cluster) + if err != nil { + return nil, err + } interval := 10 * time.Second if value, ok := feConf["cloud_cluster_check_interval_second"]; ok { seconds, err := strconv.Atoi(fmt.Sprint(value)) @@ -320,10 +322,12 @@ func (dcgs *DisaggregatedComputeGroupsController) decommissionSnapshot(ctx conte } // Allow two configured synchronization periods, and still verify FE state. snapshot.SyncWait = 2 * interval - user, password := dcgs.GetManagementAdminUserAndPWD(ctx, cluster) - tlsConfig, secretName := dcgs.DisaggregatedSubDefaultController.FindSecretTLSConfig(feConf, cluster) - secret, err := k8s.GetSecret(ctx, dcgs.K8sclient, cluster.Namespace, secretName) - if tlsConfig != nil && err != nil { + user, password, err := dcgs.GetManagementAdminUserAndPWD(ctx, cluster) + if err != nil { + return nil, err + } + tlsConfig, secret, err := sc.LoadManagementTLS(ctx, dcgs.K8sclient, cluster.Namespace, feConf, cluster.Spec.FeSpec.Secrets) + if err != nil { return nil, err } for _, fe := range frontends { diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/prepare_modify.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/prepare_modify.go index a20d9d29..75c1e1c5 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/prepare_modify.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/prepare_modify.go @@ -26,9 +26,9 @@ import ( "time" dv1 "github.com/apache/doris-operator/api/disaggregated/v1" - "github.com/apache/doris-operator/pkg/common/utils/k8s" "github.com/apache/doris-operator/pkg/common/utils/mysql" "github.com/apache/doris-operator/pkg/common/utils/resource" + sc "github.com/apache/doris-operator/pkg/controller/sub_controller" appv1 "k8s.io/api/apps/v1" "k8s.io/klog/v2" ) @@ -141,12 +141,18 @@ func dropBackendEnsuringAbsent(sqlClient *mysql.DB, target *mysql.Backend) error func (dcgs *DisaggregatedComputeGroupsController) getMasterSqlClient(ctx context.Context, cluster *dv1.DorisDisaggregatedCluster) (*mysql.DB, error) { // get user and password - adminUserName, password := dcgs.GetManagementAdminUserAndPWD(ctx, cluster) + adminUserName, password, err := dcgs.GetManagementAdminUserAndPWD(ctx, cluster) + if err != nil { + return nil, err + } // get host and port // When the operator and dcr are deployed in different namespace, it will be inaccessible, so need to add the dcr svc namespace host := cluster.GetFEVIPAddresss() - confMap := dcgs.GetConfigValuesFromConfigMaps(cluster.Namespace, resource.FE_RESOLVEKEY, cluster.Spec.FeSpec.ConfigMaps) + confMap, err := dcgs.ManagementFEConfig(ctx, cluster) + if err != nil { + return nil, err + } queryPort := resource.GetPort(confMap, resource.QUERY_PORT) // connect to doris sql to get master node @@ -161,8 +167,10 @@ func (dcgs *DisaggregatedComputeGroupsController) getMasterSqlClient(ctx context if cluster.Spec.EnableDecommission { dbConf.Timeout = 10 * time.Second } - tlsConfig, secretName := dcgs.DisaggregatedSubDefaultController.FindSecretTLSConfig(confMap, cluster) - secret, _ := k8s.GetSecret(context.Background(), dcgs.K8sclient, cluster.Namespace, secretName) + tlsConfig, secret, err := sc.LoadManagementTLS(ctx, dcgs.K8sclient, cluster.Namespace, confMap, cluster.Spec.FeSpec.Secrets) + if err != nil { + return nil, err + } // Connect to the master and run the SQL statement of system admin, because it is not excluded that the user can shrink be and fe at the same time masterDBClient, err := mysql.NewDorisMasterSqlDB(dbConf, tlsConfig, secret) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go index 168b9eb9..9e6ce3c5 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go @@ -131,6 +131,7 @@ func (dcgs *DisaggregatedComputeGroupsController) NewPodTemplateSpec(ddc *dv1.Do cgUniqueId := selector[dv1.DorisDisaggregatedComputeGroupUniqueId] pts.Spec.Affinity = dcgs.ConstructDefaultAffinity(dv1.DorisDisaggregatedComputeGroupUniqueId, cgUniqueId, pts.Spec.Affinity) + sub.SetAdminCredentialRevision(ddc, &pts) return pts } @@ -213,13 +214,7 @@ func (dcgs *DisaggregatedComputeGroupsController) newSpecificEnvs(ddc *dv1.Doris corev1.EnvVar{Name: resource.ENV_FE_ADDR, Value: feAddr}, corev1.EnvVar{Name: resource.ENV_FE_PORT, Value: fqpStr}) - // add user and password envs - if ddc.Spec.AdminUser != nil { - cgEnvs = append(cgEnvs, - corev1.EnvVar{Name: resource.ADMIN_USER, Value: ddc.Spec.AdminUser.Name}, - corev1.EnvVar{Name: resource.ADMIN_PASSWD, Value: ddc.Spec.AdminUser.Password}, - ) - } + cgEnvs = append(cgEnvs, sub.AdminCredentialEnvs(ddc)...) if cg.EnableWorkloadGroup { cgEnvs = append(cgEnvs, diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go index 6a1ef6a5..f5095e51 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go @@ -375,15 +375,23 @@ func (dfc *DisaggregatedFEController) recycleResources(ctx context.Context, ddc // dropFEBySQLClient only delete the fe nodes whose pod number is greater than the expected number (cluster.Spec.FeSpec.Replicas) by calling the drop_node interface func (dfc *DisaggregatedFEController) dropFEBySQLClient(ctx context.Context, k8sclient client.Client, cluster *v1.DorisDisaggregatedCluster) error { // get adminuserName and pwd - adminUserName, password := dfc.GetManagementAdminUserAndPWD(ctx, cluster) + adminUserName, password, err := dfc.GetManagementAdminUserAndPWD(ctx, cluster) + if err != nil { + return err + } // get host and port // When the operator and dcr are deployed in different namespace, it will be inaccessible, so need to add the dcr svc namespace host := cluster.GetFEVIPAddresss() - confMap := dfc.GetConfigValuesFromConfigMaps(cluster.Namespace, resource.FE_RESOLVEKEY, cluster.Spec.FeSpec.ConfigMaps) + confMap, err := dfc.ManagementFEConfig(ctx, cluster) + if err != nil { + return err + } queryPort := resource.GetPort(confMap, resource.QUERY_PORT) - tlsConfig, secretName := dfc.DisaggregatedSubDefaultController.FindSecretTLSConfig(confMap, cluster) - secret, _ := k8s.GetSecret(context.Background(), dfc.K8sclient, cluster.Namespace, secretName) + tlsConfig, secret, err := sc.LoadManagementTLS(ctx, dfc.K8sclient, cluster.Namespace, confMap, cluster.Spec.FeSpec.Secrets) + if err != nil { + return err + } // connect to doris sql to get master node // It may not be the master, or even the node that needs to be deleted, causing the deletion SQL to fail. diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go index b97cf44e..d9a66cb1 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go @@ -119,6 +119,7 @@ func (dfc *DisaggregatedFEController) NewPodTemplateSpec(ddc *v1.DorisDisaggrega pts.Spec.Affinity = dfc.ConstructDefaultAffinity(v1.DorisDisaggregatedClusterName, labels[v1.DorisDisaggregatedClusterName], ddc.Spec.FeSpec.Affinity) + sub.SetAdminCredentialRevision(ddc, &pts) return pts } @@ -180,13 +181,7 @@ func (dfc *DisaggregatedFEController) newSpecificEnvs(ddc *v1.DorisDisaggregated corev1.EnvVar{Name: resource.ENV_FE_ELECT_NUMBER, Value: strconv.FormatInt(int64(ddc.GetElectionNumber()), 10)}, ) - // add user and password envs - if ddc.Spec.AdminUser != nil { - feEnvs = append(feEnvs, - corev1.EnvVar{Name: resource.ADMIN_USER, Value: ddc.Spec.AdminUser.Name}, - corev1.EnvVar{Name: resource.ADMIN_PASSWD, Value: ddc.Spec.AdminUser.Password}, - ) - } + feEnvs = append(feEnvs, sub.AdminCredentialEnvs(ddc)...) return feEnvs } diff --git a/pkg/controller/sub_controller/disaggregated_subcontroller.go b/pkg/controller/sub_controller/disaggregated_subcontroller.go index da3afa61..dc6faa0f 100644 --- a/pkg/controller/sub_controller/disaggregated_subcontroller.go +++ b/pkg/controller/sub_controller/disaggregated_subcontroller.go @@ -21,15 +21,12 @@ import ( "context" "encoding/json" "fmt" - "path" - "path/filepath" "strconv" "strings" "github.com/apache/doris-operator/api/disaggregated/v1" "github.com/apache/doris-operator/pkg/common/utils/k8s" "github.com/apache/doris-operator/pkg/common/utils/metadata" - "github.com/apache/doris-operator/pkg/common/utils/mysql" "github.com/apache/doris-operator/pkg/common/utils/resource" "github.com/apache/doris-operator/pkg/common/utils/set" appv1 "k8s.io/api/apps/v1" @@ -104,6 +101,20 @@ func (d *DisaggregatedSubDefaultController) GetConfigValuesFromConfigMaps(namesp return nil } +// ManagementFEConfig propagates configuration read errors before choosing SQL transport. +func (d *DisaggregatedSubDefaultController) ManagementFEConfig(ctx context.Context, ddc *v1.DorisDisaggregatedCluster) (map[string]interface{}, error) { + for _, ref := range ddc.Spec.FeSpec.ConfigMaps { + cm, err := k8s.GetConfigMap(ctx, d.K8sclient, ddc.Namespace, ref.Name) + if err != nil { + return nil, err + } + if content, ok := cm.Data[resource.FE_RESOLVEKEY]; ok { + return resource.ParseDorisConfig([]byte(content), resource.DEFAULT_ROOT_PATH+"/fe", false) + } + } + return nil, nil +} + func (d *DisaggregatedSubDefaultController) resolveStartConfig(vb []byte, resolveKey string) map[string]interface{} { var home string switch resolveKey { @@ -283,19 +294,22 @@ func (d *DisaggregatedSubDefaultController) RestrictConditionsEqual(new *appv1.S } } -func (d *DisaggregatedSubDefaultController) GetManagementAdminUserAndPWD(ctx context.Context, ddc *v1.DorisDisaggregatedCluster) (string, string) { - adminUserName := "root" - password := "" - if ddc.Spec.AuthSecret != "" { - secret, _ := k8s.GetSecret(ctx, d.K8sclient, ddc.Namespace, ddc.Spec.AuthSecret) - adminUserName, password = resource.GetDorisLoginInformation(secret) - } else if ddc.Spec.AdminUser != nil { - adminUserName = ddc.Spec.AdminUser.Name - password = ddc.Spec.AdminUser.Password +func (d *DisaggregatedSubDefaultController) GetManagementAdminUserAndPWD(ctx context.Context, ddc *v1.DorisDisaggregatedCluster) (string, string, error) { + if name := ddc.Spec.AuthSecret; name != "" { + secret, err := k8s.GetSecret(ctx, d.K8sclient, ddc.Namespace, name) + if err != nil { + return "", "", fmt.Errorf("read management Secret %s/%s: %w", ddc.Namespace, name, err) + } + if _, ok := secret.Data[corev1.BasicAuthPasswordKey]; !ok { + return "", "", fmt.Errorf("management Secret %s/%s lacks password key", ddc.Namespace, name) + } + user, password := resource.GetDorisLoginInformation(secret) + return user, password, nil } - - return adminUserName, password - + if user := ddc.Spec.AdminUser; user != nil { + return user.Name, user.Password, nil + } + return "root", "", nil } // add cluster specification on container spec. this is useful to add common spec on different type pods, example: kerberos volume for fe and be. @@ -859,34 +873,3 @@ func (d *DisaggregatedSubDefaultController) getFEMetaPath(confMap map[string]int } return v.(string) } - -func (d *DisaggregatedSubDefaultController) FindSecretTLSConfig(feConfMap map[string]interface{}, ddc *v1.DorisDisaggregatedCluster) (*mysql.TLSConfig, string /*secret name*/) { - enableTLS := resource.GetString(feConfMap, resource.ENABLE_TLS_KEY) - if enableTLS == "" { - return nil, "" - } - - caCertFile := resource.GetString(feConfMap, resource.TLS_CA_CERTIFICATE_PATH_KEY) - clientCertFile := resource.GetString(feConfMap, resource.TLS_CERTIFICATE_PATH_KEY) - clientKeyFile := resource.GetString(feConfMap, resource.TLS_PRIVATE_KEY_PATH_KEY) - caFileName := path.Base(caCertFile) - clientCertFileName := path.Base(clientCertFile) - clientKeyFileName := path.Base(clientKeyFile) - - caCertDir := filepath.Dir(caCertFile) - secretName := "" - for _, sn := range ddc.Spec.FeSpec.Secrets { - if sn.MountPath == caCertDir { - secretName = sn.SecretName - break - } - } - - tlsConfig := &mysql.TLSConfig{ - CAFileName: caFileName, - ClientCertFileName: clientCertFileName, - ClientKeyFileName: clientKeyFileName, - } - - return tlsConfig, secretName -} diff --git a/pkg/controller/sub_controller/fe/prepare_modify.go b/pkg/controller/sub_controller/fe/prepare_modify.go index e4ab34ea..7587a6b7 100644 --- a/pkg/controller/sub_controller/fe/prepare_modify.go +++ b/pkg/controller/sub_controller/fe/prepare_modify.go @@ -25,7 +25,6 @@ import ( "github.com/apache/doris-operator/pkg/common/utils/resource" sc "github.com/apache/doris-operator/pkg/controller/sub_controller" appv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -111,7 +110,10 @@ func (fc *Controller) dropObserverBySqlClient(ctx context.Context, k8sclient cli serviceName := v1.GenerateExternalServiceName(targetDCR, v1.Component_FE) // When the operator and dcr are deployed in different namespace, it will be inaccessible, so need to add the dcr svc namespace host := serviceName + "." + targetDCR.Namespace - maps, _ := k8s.GetConfig(ctx, k8sclient, &targetDCR.Spec.FeSpec.ConfigMapInfo, targetDCR.Namespace, v1.Component_FE) + maps, err := k8s.GetConfig(ctx, k8sclient, &targetDCR.Spec.FeSpec.ConfigMapInfo, targetDCR.Namespace, v1.Component_FE) + if err != nil { + return err + } queryPort := resource.GetPort(maps, resource.QUERY_PORT) // connect to doris sql to get master node @@ -125,10 +127,9 @@ func (fc *Controller) dropObserverBySqlClient(ctx context.Context, k8sclient cli } // check if TLS is enabled in FE config and find the corresponding secret - tlsConfig, secretName := fc.FindSecretTLSConfig(maps, targetDCR) - var tlsSecret *corev1.Secret - if tlsConfig != nil && secretName != "" { - tlsSecret, _ = k8s.GetSecret(ctx, k8sclient, targetDCR.Namespace, secretName) + tlsConfig, tlsSecret, err := sc.LoadManagementTLS(ctx, k8sclient, targetDCR.Namespace, maps, targetDCR.Spec.FeSpec.Secrets) + if err != nil { + return err } masterDBClient, err := mysql.NewDorisMasterSqlDB(dbConf, tlsConfig, tlsSecret) diff --git a/pkg/controller/sub_controller/management_credentials_test.go b/pkg/controller/sub_controller/management_credentials_test.go new file mode 100644 index 00000000..74567358 --- /dev/null +++ b/pkg/controller/sub_controller/management_credentials_test.go @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sub_controller + +import ( + "context" + "testing" + + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + "github.com/apache/doris-operator/pkg/common/utils/mysql" + "github.com/apache/doris-operator/pkg/common/utils/resource" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestManagementTLS(t *testing.T) { + ctx := context.Background() + c := fake.NewClientBuilder().Build() + conf := map[string]interface{}{resource.TLS_CA_CERTIFICATE_PATH_KEY: "/etc/doris/ca.pem", resource.TLS_CERTIFICATE_PATH_KEY: "/etc/doris/client.pem", resource.TLS_PRIVATE_KEY_PATH_KEY: "/etc/doris/client.key"} + mounts := []dv1.Secret{{SecretName: "certs"}} + for _, val := range []string{"", "false", "0"} { + conf[resource.ENABLE_TLS_KEY] = val + cfg, secret, err := LoadManagementTLS(ctx, c, "ns", conf, mounts) + if err != nil || cfg != nil || secret != nil { + t.Fatalf("disabled TLS %q: %v", val, err) + } + } + conf[resource.ENABLE_TLS_KEY] = "invalid" + if _, _, err := LoadManagementTLS(ctx, c, "ns", conf, mounts); err == nil { + t.Fatal("invalid boolean accepted") + } + conf[resource.ENABLE_TLS_KEY] = "true" + for _, mount := range [][]dv1.Secret{nil, mounts} { + if _, _, err := LoadManagementTLS(ctx, c, "ns", conf, mount); err == nil { + t.Fatal("missing TLS Secret accepted") + } + } + if err := c.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "certs"}}); err != nil { + t.Fatal(err) + } + cfg, secret, err := LoadManagementTLS(ctx, c, "ns", conf, mounts) + if err != nil || cfg.CAFileName != "ca.pem" || secret.Name != "certs" { + t.Fatalf("recover after Secret creation: %v", err) + } + for _, material := range []*corev1.Secret{nil, secret} { + if _, err := mysql.NewDorisSqlDB(mysql.DBConfig{}, cfg, material); err == nil { + t.Fatal("incomplete key material accepted") + } + } +} + +func TestExplicitManagementSecret(t *testing.T) { + ctx := context.Background() + c := fake.NewClientBuilder().Build() + controller := DisaggregatedSubDefaultController{K8sclient: c} + ddc := &dv1.DorisDisaggregatedCluster{} + ddc.Namespace = "ns" + check := func(wantUser, wantPass string, wantErr bool) { + t.Helper() + user, pass, err := controller.GetManagementAdminUserAndPWD(ctx, ddc) + if (err != nil) != wantErr || user != wantUser || pass != wantPass { + t.Fatalf("unexpected credentials or error: %v", err) + } + } + check("root", "", false) + ddc.Spec.AdminUser = &dv1.AdminUser{Name: "legacy", Password: "old"} + check("legacy", "old", false) + ddc.Spec.AuthSecret = "account" + check("", "", true) + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "account"}} + if err := c.Create(ctx, secret); err != nil { + t.Fatal(err) + } + check("", "", true) + secret.Data = map[string][]byte{"password": []byte("")} + if err := c.Update(ctx, secret); err != nil { + t.Fatal(err) + } + check("root", "", false) + secret.Data = map[string][]byte{"username": []byte("managed"), "password": []byte("new")} + if err := c.Update(ctx, secret); err != nil { + t.Fatal(err) + } + check("managed", "new", false) +} + +func TestAdminCredentialMigration(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + for _, register := range []func(*runtime.Scheme) error{dv1.AddToScheme, appsv1.AddToScheme, corev1.AddToScheme} { + if err := register(scheme); err != nil { + t.Fatal(err) + } + } + for _, existing := range []bool{false, true} { + ddc := &dv1.DorisDisaggregatedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns", UID: "cluster-uid"}} + ddc.Spec.AdminUser = &dv1.AdminUser{Name: "dbuser", Password: "first"} + ddc.Spec.AuthSecret = "separate-account" + objects := []client.Object{ddc} + if existing { + objects = append(objects, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "legacy", Namespace: "ns", OwnerReferences: []metav1.OwnerReference{{UID: ddc.UID}}}}) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + if err := c.Get(ctx, client.ObjectKeyFromObject(ddc), ddc); err != nil { + t.Fatal(err) + } + ensure := func() { + t.Helper() + if err := EnsureAdminCredentials(ctx, c, ddc); err != nil { + t.Fatal(err) + } + } + ensure() + if existing { + if ddc.Annotations[adminCredentialMode] != modeLegacy || AdminCredentialEnvs(ddc)[1].Value != "first" { + t.Fatal("upgrade changed existing environment") + } + before := ddc.DeepCopy() + ddc.Annotations[adminCredentialMode] = modeSecret + if err := c.Patch(ctx, ddc, client.MergeFrom(before)); err != nil { + t.Fatal(err) + } + ensure() + } + env := AdminCredentialEnvs(ddc)[1] + if env.Value != "" || env.ValueFrom.SecretKeyRef.Name != adminSecretName(ddc) || env.ValueFrom.SecretKeyRef.Key != "password" { + t.Fatal("incorrect Secret reference") + } + rev := ddc.Annotations[adminCredentialRevision] + ensure() + if rev == "" || rev != ddc.Annotations[adminCredentialRevision] { + t.Fatal("credential revision is unstable") + } + ddc.Spec.AdminUser.Password = "second" + ensure() + var secret corev1.Secret + if err := c.Get(ctx, client.ObjectKey{Namespace: "ns", Name: adminSecretName(ddc)}, &secret); err != nil { + t.Fatal(err) + } + if string(secret.Data["password"]) != "second" || rev == ddc.Annotations[adminCredentialRevision] || !metav1.IsControlledBy(&secret, ddc) { + t.Fatal("credential rotation failed") + } + secret.OwnerReferences = nil + if err := c.Update(ctx, &secret); err != nil { + t.Fatal(err) + } + if err := EnsureAdminCredentials(ctx, c, ddc); err == nil { + t.Fatal("accepted a Secret owned elsewhere") + } + } +} diff --git a/pkg/controller/sub_controller/management_tls.go b/pkg/controller/sub_controller/management_tls.go new file mode 100644 index 00000000..8f37d38b --- /dev/null +++ b/pkg/controller/sub_controller/management_tls.go @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sub_controller + +import ( + "context" + "fmt" + "path" + "strconv" + "strings" + + dv1 "github.com/apache/doris-operator/api/disaggregated/v1" + dorisv1 "github.com/apache/doris-operator/api/doris/v1" + "github.com/apache/doris-operator/pkg/common/utils/k8s" + "github.com/apache/doris-operator/pkg/common/utils/mysql" + "github.com/apache/doris-operator/pkg/common/utils/resource" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// LoadManagementTLS resolves the certificate mount before opening a SQL connection. +func LoadManagementTLS[S dorisv1.Secret | dv1.Secret](ctx context.Context, c client.Client, namespace string, conf map[string]interface{}, mounts []S) (*mysql.TLSConfig, *corev1.Secret, error) { + raw, ok := conf[resource.ENABLE_TLS_KEY] + if !ok || raw == "" { + return nil, nil, nil + } + enabled, err := strconv.ParseBool(strings.TrimSpace(fmt.Sprint(raw))) + if err != nil { + return nil, nil, fmt.Errorf("invalid enable_tls setting: %w", err) + } + if !enabled { + return nil, nil, nil + } + ca := resource.GetString(conf, resource.TLS_CA_CERTIFICATE_PATH_KEY) + cert := resource.GetString(conf, resource.TLS_CERTIFICATE_PATH_KEY) + key := resource.GetString(conf, resource.TLS_PRIVATE_KEY_PATH_KEY) + if ca == "" || cert == "" || key == "" { + return nil, nil, fmt.Errorf("management TLS requires CA, client certificate and key paths") + } + for _, item := range mounts { + mount := dorisv1.Secret(item) + dir := mount.MountPath + if dir == "" { + dir = resource.ConfigEnvPath + } + if path.Clean(dir) != path.Dir(ca) { + continue + } + secret, err := k8s.GetSecret(ctx, c, namespace, mount.SecretName) + if err != nil { + return nil, nil, fmt.Errorf("read management TLS Secret %s/%s: %w", namespace, mount.SecretName, err) + } + return &mysql.TLSConfig{CAFileName: path.Base(ca), ClientCertFileName: path.Base(cert), ClientKeyFileName: path.Base(key)}, secret, nil + } + return nil, nil, fmt.Errorf("no management TLS Secret mounted at %s", path.Dir(ca)) +} diff --git a/pkg/controller/sub_controller/sub_controller.go b/pkg/controller/sub_controller/sub_controller.go index feb36c7d..0854f7f0 100644 --- a/pkg/controller/sub_controller/sub_controller.go +++ b/pkg/controller/sub_controller/sub_controller.go @@ -23,7 +23,6 @@ import ( dorisv1 "github.com/apache/doris-operator/api/doris/v1" utils "github.com/apache/doris-operator/pkg/common/utils" "github.com/apache/doris-operator/pkg/common/utils/k8s" - "github.com/apache/doris-operator/pkg/common/utils/mysql" "github.com/apache/doris-operator/pkg/common/utils/resource" "github.com/apache/doris-operator/pkg/common/utils/set" appv1 "k8s.io/api/apps/v1" @@ -33,8 +32,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" - "path" - "path/filepath" "sigs.k8s.io/controller-runtime/pkg/client" "strconv" "strings" @@ -287,41 +284,6 @@ func (d *SubDefaultController) CheckSecretExist(ctx context.Context, dcr *dorisv } } -// FindSecretTLSConfig reads TLS configuration from FE config map and returns -// the TLS config and secret name for establishing TLS-enabled MySQL connections. -func (d *SubDefaultController) FindSecretTLSConfig(feConfMap map[string]interface{}, dcr *dorisv1.DorisCluster) (*mysql.TLSConfig, string) { - enableTLS := resource.GetString(feConfMap, resource.ENABLE_TLS_KEY) - if enableTLS == "" { - return nil, "" - } - - caCertFile := resource.GetString(feConfMap, resource.TLS_CA_CERTIFICATE_PATH_KEY) - clientCertFile := resource.GetString(feConfMap, resource.TLS_CERTIFICATE_PATH_KEY) - clientKeyFile := resource.GetString(feConfMap, resource.TLS_PRIVATE_KEY_PATH_KEY) - caFileName := path.Base(caCertFile) - clientCertFileName := path.Base(clientCertFile) - clientKeyFileName := path.Base(clientKeyFile) - - caCertDir := filepath.Dir(caCertFile) - secretName := "" - if dcr.Spec.FeSpec != nil { - for _, sn := range dcr.Spec.FeSpec.Secrets { - if sn.MountPath == caCertDir { - secretName = sn.SecretName - break - } - } - } - - tlsConfig := &mysql.TLSConfig{ - CAFileName: caFileName, - ClientCertFileName: clientCertFileName, - ClientKeyFileName: clientKeyFileName, - } - - return tlsConfig, secretName -} - // CheckSharedPVC verifies two points: // 1. Whether the SharePVC exists // 2. Whether the AccessMode of the SharePVC is ReadWriteMany