Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions config/operator/disaggregated-operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ rules:
resources:
- secrets
verbs:
- create
- get
- update
- list
Expand Down
1 change: 1 addition & 0 deletions config/operator/operator-sign-ns.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ rules:
resources:
- secrets
verbs:
- create
- get
- update
- list
Expand Down
1 change: 1 addition & 0 deletions config/operator/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ rules:
resources:
- secrets
verbs:
- create
- get
- update
- list
Expand Down
2 changes: 2 additions & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,10 @@ rules:
resources:
- secrets
verbs:
- create
- get
- list
- update
- watch
- apiGroups:
- ""
Expand Down
13 changes: 12 additions & 1 deletion doc/examples/disaggregated/cluster/ddc-pwd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,15 @@ spec:
image: apache/doris:be-3.0.3
- uniqueId: cg2
replicas: 3
image: apache/doris:be-3.0.3
image: apache/doris:be-3.0.3

# New clusters receive the adminUser environment variables from an owned Secret.
# On upgrade, existing clusters keep their current credential delivery. To migrate
# during a maintenance window, apply the updated Secret RBAC and verify that
# the operator has its usual compute-group rollout permissions in the managed
# namespace (create pods/exec and delete pods, as in disaggregated-operator.yaml).
# Annotate the DDC and wait for FE and compute-group rollouts:
# kubectl annotate ddc <name> -n <namespace> doris.apache.org/admin-credential-mode=secret --overwrite
# The adminUser fields remain in the DDC. Use spec.authSecret alone to keep
# credentials outside the DDC. If both are configured, the environment still
# uses adminUser and the existing authSecret mount is preserved.
1 change: 1 addition & 0 deletions helm-charts/doris-operator/templates/clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ rules:
resources:
- secrets
verbs:
- create
- get
- update
- list
Expand Down
5 changes: 4 additions & 1 deletion pkg/common/utils/mysql/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ func NewDorisSqlDB(cfg DBConfig, tlsConfig *TLSConfig, secret *corev1.Secret) (*
}
rootCertPool := x509.NewCertPool()

if tlsConfig != nil && secret != nil {
if tlsConfig != nil {
if secret == nil {
return nil, errors.New("management TLS requires a certificate Secret")
}
ca := secret.Data[tlsConfig.CAFileName]
clientCert := secret.Data[tlsConfig.ClientCertFileName]
clientKey := secret.Data[tlsConfig.ClientKeyFileName]
Expand Down
6 changes: 6 additions & 0 deletions pkg/controller/disaggregated_cluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ func fdbClusterFile(obj client.Object) (string, bool) {

func (dc *DisaggregatedClusterReconciler) resourceBuilder(builder *ctrl.Builder) *ctrl.Builder {
return builder.For(&dv1.DorisDisaggregatedCluster{}).
Owns(&corev1.Secret{}).
Owns(&appv1.StatefulSet{}, controller_builder.MatchEveryOwner).
Owns(&corev1.Service{}, controller_builder.MatchEveryOwner)
}
Expand Down Expand Up @@ -234,6 +235,11 @@ func (dc *DisaggregatedClusterReconciler) Reconcile(ctx context.Context, req rec
return ctrl.Result{}, err
}

if err := sc.EnsureAdminCredentials(ctx, dc.Client, &ddc); err != nil {
dc.Recorder.Event(&ddc, corev1.EventTypeWarning, "AdminCredentialsPending", err.Error())
return ctrl.Result{}, err
}

hv := hash.HashObject(ddc.Spec)

var res ctrl.Result
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/doriscluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ var (
//+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get
//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch
//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update
//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
Expand Down
168 changes: 168 additions & 0 deletions pkg/controller/sub_controller/admin_credentials.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package sub_controller

import (
"context"
"fmt"
"reflect"

dv1 "github.com/apache/doris-operator/api/disaggregated/v1"
"github.com/apache/doris-operator/pkg/common/utils/resource"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
adminCredentialMode = "doris.apache.org/admin-credential-mode"
adminCredentialRevision = "doris.apache.org/admin-credential-revision"
modeSecret = "secret"
modeLegacy = "legacy"
)

func adminSecretName(ddc *dv1.DorisDisaggregatedCluster) string {
return "doris-admin-" + string(ddc.UID)
}

// EnsureAdminCredentials establishes the cluster-wide mode before any child is rendered.
func EnsureAdminCredentials(ctx context.Context, c client.Client, ddc *dv1.DorisDisaggregatedCluster) error {
if !ddc.DeletionTimestamp.IsZero() {
return nil
}
mode := ddc.Annotations[adminCredentialMode]
if mode == "" {
existing, err := hasExistingDeployment(ctx, c, ddc)
if err != nil {
return err
}
mode = modeSecret
if existing {
mode = modeLegacy
}
}
if mode != modeSecret && mode != modeLegacy {
return fmt.Errorf("unknown %s value %q", adminCredentialMode, mode)
}
revision := ""
if mode == modeSecret && ddc.Spec.AdminUser != nil {
secret, err := reconcileAdminSecret(ctx, c, ddc)
if err != nil {
return err
}
revision = secret.Annotations[adminCredentialRevision]
}
if mode == ddc.Annotations[adminCredentialMode] && revision == ddc.Annotations[adminCredentialRevision] {
return nil
}
before := ddc.DeepCopy()
if ddc.Annotations == nil {
ddc.Annotations = make(map[string]string)
}
ddc.Annotations[adminCredentialMode] = mode
if revision == "" {
delete(ddc.Annotations, adminCredentialRevision)
} else {
ddc.Annotations[adminCredentialRevision] = revision
}
return c.Patch(ctx, ddc, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}))
}

func hasExistingDeployment(ctx context.Context, c client.Client, ddc *dv1.DorisDisaggregatedCluster) (bool, error) {
if !reflect.ValueOf(ddc.Status).IsZero() {
return true, nil
}
var workloads appsv1.StatefulSetList
if err := c.List(ctx, &workloads, client.InNamespace(ddc.Namespace)); err != nil {
return false, err
}
for _, st := range workloads.Items {
// Older releases created non-controller owner references.
for _, owner := range st.OwnerReferences {
if ddc.UID != "" && owner.UID == ddc.UID {
return true, nil
}
}
}
return false, nil
}

func reconcileAdminSecret(ctx context.Context, c client.Client, ddc *dv1.DorisDisaggregatedCluster) (*corev1.Secret, error) {
name := adminSecretName(ddc)
data := map[string][]byte{corev1.BasicAuthUsernameKey: []byte(ddc.Spec.AdminUser.Name), corev1.BasicAuthPasswordKey: []byte(ddc.Spec.AdminUser.Password)}
secret := &corev1.Secret{}
err := c.Get(ctx, client.ObjectKey{Namespace: ddc.Namespace, Name: name}, secret)
if apierrors.IsNotFound(err) {
owner := metav1.NewControllerRef(ddc, dv1.GroupVersion.WithKind("DorisDisaggregatedCluster"))
owner.BlockOwnerDeletion = nil
secret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: ddc.Namespace, Name: name,
OwnerReferences: []metav1.OwnerReference{*owner},
Annotations: map[string]string{adminCredentialRevision: string(uuid.NewUUID())},
},
Data: data,
}
return secret, c.Create(ctx, secret)
}
if err != nil {
return nil, err
}
if !metav1.IsControlledBy(secret, ddc) {
return nil, fmt.Errorf("Secret %s/%s already exists with a different owner", ddc.Namespace, name)
}
if secret.DeletionTimestamp != nil {
return nil, fmt.Errorf("waiting for deletion of Secret %s/%s", ddc.Namespace, name)
}
// A metadata-only edit must not restart workloads that consume this Secret.
if reflect.DeepEqual(secret.Data, data) && secret.Annotations[adminCredentialRevision] != "" {
return secret, nil
}
secret.Data = data
if secret.Annotations == nil {
secret.Annotations = make(map[string]string)
}
secret.Annotations[adminCredentialRevision] = string(uuid.NewUUID())
return secret, c.Update(ctx, secret)
}

func AdminCredentialEnvs(ddc *dv1.DorisDisaggregatedCluster) []corev1.EnvVar {
user := ddc.Spec.AdminUser
if user == nil {
return nil
}
if ddc.Annotations[adminCredentialMode] != modeSecret {
return []corev1.EnvVar{{Name: resource.ADMIN_USER, Value: user.Name}, {Name: resource.ADMIN_PASSWD, Value: user.Password}}
}
ref := func(key string) *corev1.EnvVarSource {
return &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: adminSecretName(ddc)}, Key: key}}
}
return []corev1.EnvVar{{Name: resource.ADMIN_USER, ValueFrom: ref(corev1.BasicAuthUsernameKey)}, {Name: resource.ADMIN_PASSWD, ValueFrom: ref(corev1.BasicAuthPasswordKey)}}
}

// Secret-backed environment variables are read once when the container starts.
func SetAdminCredentialRevision(ddc *dv1.DorisDisaggregatedCluster, pts *corev1.PodTemplateSpec) {
if rev := ddc.Annotations[adminCredentialRevision]; rev != "" && ddc.Annotations[adminCredentialMode] == modeSecret && ddc.Spec.AdminUser != nil {
if pts.Annotations == nil {
pts.Annotations = make(map[string]string)
}
pts.Annotations[adminCredentialRevision] = rev
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -824,21 +824,30 @@ func (dcgs *DisaggregatedComputeGroupsController) UpdateComponentStatus(obj clie

func (dcgs *DisaggregatedComputeGroupsController) recordComputeGroupIds(ddc *dv1.DorisDisaggregatedCluster) error {
// get user and password
adminUserName, password := dcgs.GetManagementAdminUserAndPWD(context.Background(), ddc)
ctx := context.Background()
adminUserName, password, err := dcgs.GetManagementAdminUserAndPWD(ctx, ddc)
if err != nil {
return err
}

// get host and port
// When the operator and dcr are deployed in different namespace, it will be inaccessible, so need to add the dcr svc namespace
host := ddc.GetFEVIPAddresss()
confMap := dcgs.GetConfigValuesFromConfigMaps(ddc.Namespace, resource.FE_RESOLVEKEY, ddc.Spec.FeSpec.ConfigMaps)
confMap, err := dcgs.ManagementFEConfig(ctx, ddc)
if err != nil {
return err
}
queryPort := resource.GetPort(confMap, resource.QUERY_PORT)
cfg := mysql.NewDBConfig()
cfg.User = adminUserName
cfg.Password = password
cfg.Host = host
cfg.Port = strconv.FormatInt(int64(queryPort), 10)

tlsConfig, secretName := dcgs.DisaggregatedSubDefaultController.FindSecretTLSConfig(confMap, ddc)
secret, _ := k8s.GetSecret(context.Background(), dcgs.K8sclient, ddc.Namespace, secretName)
tlsConfig, secret, err := sc.LoadManagementTLS(ctx, dcgs.K8sclient, ddc.Namespace, confMap, ddc.Spec.FeSpec.Secrets)
if err != nil {
return err
}

db, err := mysql.NewDorisSqlDB(cfg, tlsConfig, secret)
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"time"

dv1 "github.com/apache/doris-operator/api/disaggregated/v1"
"github.com/apache/doris-operator/pkg/common/utils/k8s"
ms "github.com/apache/doris-operator/pkg/common/utils/metaservice"
"github.com/apache/doris-operator/pkg/common/utils/mysql"
"github.com/apache/doris-operator/pkg/common/utils/resource"
Expand Down Expand Up @@ -307,7 +306,10 @@ func (dcgs *DisaggregatedComputeGroupsController) decommissionSnapshot(ctx conte
if len(frontends) == 0 {
return nil, errors.New("no FE observations are available")
}
feConf := dcgs.GetConfigValuesFromConfigMaps(cluster.Namespace, resource.FE_RESOLVEKEY, cluster.Spec.FeSpec.ConfigMaps)
feConf, err := dcgs.ManagementFEConfig(ctx, cluster)
if err != nil {
return nil, err
}
interval := 10 * time.Second
if value, ok := feConf["cloud_cluster_check_interval_second"]; ok {
seconds, err := strconv.Atoi(fmt.Sprint(value))
Expand All @@ -320,10 +322,12 @@ func (dcgs *DisaggregatedComputeGroupsController) decommissionSnapshot(ctx conte
}
// Allow two configured synchronization periods, and still verify FE state.
snapshot.SyncWait = 2 * interval
user, password := dcgs.GetManagementAdminUserAndPWD(ctx, cluster)
tlsConfig, secretName := dcgs.DisaggregatedSubDefaultController.FindSecretTLSConfig(feConf, cluster)
secret, err := k8s.GetSecret(ctx, dcgs.K8sclient, cluster.Namespace, secretName)
if tlsConfig != nil && err != nil {
user, password, err := dcgs.GetManagementAdminUserAndPWD(ctx, cluster)
if err != nil {
return nil, err
}
tlsConfig, secret, err := sc.LoadManagementTLS(ctx, dcgs.K8sclient, cluster.Namespace, feConf, cluster.Spec.FeSpec.Secrets)
if err != nil {
return nil, err
}
for _, fe := range frontends {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ import (
"time"

dv1 "github.com/apache/doris-operator/api/disaggregated/v1"
"github.com/apache/doris-operator/pkg/common/utils/k8s"
"github.com/apache/doris-operator/pkg/common/utils/mysql"
"github.com/apache/doris-operator/pkg/common/utils/resource"
sc "github.com/apache/doris-operator/pkg/controller/sub_controller"
appv1 "k8s.io/api/apps/v1"
"k8s.io/klog/v2"
)
Expand Down Expand Up @@ -141,12 +141,18 @@ func dropBackendEnsuringAbsent(sqlClient *mysql.DB, target *mysql.Backend) error

func (dcgs *DisaggregatedComputeGroupsController) getMasterSqlClient(ctx context.Context, cluster *dv1.DorisDisaggregatedCluster) (*mysql.DB, error) {
// get user and password
adminUserName, password := dcgs.GetManagementAdminUserAndPWD(ctx, cluster)
adminUserName, password, err := dcgs.GetManagementAdminUserAndPWD(ctx, cluster)
if err != nil {
return nil, err
}

// get host and port
// When the operator and dcr are deployed in different namespace, it will be inaccessible, so need to add the dcr svc namespace
host := cluster.GetFEVIPAddresss()
confMap := dcgs.GetConfigValuesFromConfigMaps(cluster.Namespace, resource.FE_RESOLVEKEY, cluster.Spec.FeSpec.ConfigMaps)
confMap, err := dcgs.ManagementFEConfig(ctx, cluster)
if err != nil {
return nil, err
}
queryPort := resource.GetPort(confMap, resource.QUERY_PORT)

// connect to doris sql to get master node
Expand All @@ -161,8 +167,10 @@ func (dcgs *DisaggregatedComputeGroupsController) getMasterSqlClient(ctx context
if cluster.Spec.EnableDecommission {
dbConf.Timeout = 10 * time.Second
}
tlsConfig, secretName := dcgs.DisaggregatedSubDefaultController.FindSecretTLSConfig(confMap, cluster)
secret, _ := k8s.GetSecret(context.Background(), dcgs.K8sclient, cluster.Namespace, secretName)
tlsConfig, secret, err := sc.LoadManagementTLS(ctx, dcgs.K8sclient, cluster.Namespace, confMap, cluster.Spec.FeSpec.Secrets)
if err != nil {
return nil, err
}

// Connect to the master and run the SQL statement of system admin, because it is not excluded that the user can shrink be and fe at the same time
masterDBClient, err := mysql.NewDorisMasterSqlDB(dbConf, tlsConfig, secret)
Expand Down
Loading
Loading