From 6eec00ce5578bfd3509e12b9da73c5d6dd33848c Mon Sep 17 00:00:00 2001 From: hechao Date: Tue, 10 Sep 2024 17:03:29 +0800 Subject: [PATCH 1/4] multi secret --- api/doris/v1/types.go | 18 ++++ api/doris/v1/zz_generated.deepcopy.go | 20 +++++ config/crd/bases/crds.yaml | 85 ++++++++++++++++--- ....doris.com_dorisdisaggregatedclusters.yaml | 21 ++--- .../doris.selectdb.com_dorisclusters.yaml | 64 ++++++++++++++ doc/examples/doriscluster-sample-secret.yaml | 55 ++++++++++++ .../doris.selectdb.com_dorisclusters.yaml | 64 ++++++++++++++ pkg/common/utils/k8s/client.go | 14 +++ pkg/common/utils/resource/pod.go | 59 +++++++++++++ .../sub_controller/be/controller.go | 2 + .../sub_controller/broker/controller.go | 2 + .../sub_controller/cn/controller.go | 2 + .../computeclusters/controller.go | 2 +- pkg/controller/sub_controller/events.go | 1 + .../sub_controller/fe/controller.go | 2 + .../sub_controller/sub_controller.go | 32 +++++++ 16 files changed, 416 insertions(+), 27 deletions(-) create mode 100644 doc/examples/doriscluster-sample-secret.yaml diff --git a/api/doris/v1/types.go b/api/doris/v1/types.go index b8df6b69..90536eb8 100644 --- a/api/doris/v1/types.go +++ b/api/doris/v1/types.go @@ -208,6 +208,10 @@ type BaseSpec struct { //Security context for all containers running in the pod (unless they override it). //+optional ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` + + // Multi Secret for pod. + // +optional + Secrets []Secret `json:"secrets,omitempty"` } type SystemInitialization struct { @@ -281,6 +285,20 @@ type MountConfigMapInfo struct { MountPath string `json:"mountPath,omitempty"` } +type Secret struct { + // name of secret that needs to mount. + SecretName string `json:"secretName,omitempty"` + + // Current Secret Mount Path, default is "/etc/doris" + // +optional + MountPath string `json:"mountPath,omitempty"` + + // Mounted read-only if true, read-write otherwise (false or unspecified). + // Defaults to false. + // +optional + ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,2,opt,name=readOnly"` +} + // ExportService consisting of expose ports for user access to software service. type ExportService struct { //type of service,the possible value for the service type are : ClusterIP, NodePort, LoadBalancer,ExternalName. diff --git a/api/doris/v1/zz_generated.deepcopy.go b/api/doris/v1/zz_generated.deepcopy.go index 9cb4938b..31569dd3 100644 --- a/api/doris/v1/zz_generated.deepcopy.go +++ b/api/doris/v1/zz_generated.deepcopy.go @@ -161,6 +161,11 @@ func (in *BaseSpec) DeepCopyInto(out *BaseSpec) { *out = new(corev1.SecurityContext) (*in).DeepCopyInto(*out) } + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]Secret, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BaseSpec. @@ -886,6 +891,21 @@ func (in *ResourceMetricSource) DeepCopy() *ResourceMetricSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Secret) DeepCopyInto(out *Secret) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Secret. +func (in *Secret) DeepCopy() *Secret { + if in == nil { + return nil + } + out := new(Secret) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SystemInitialization) DeepCopyInto(out *SystemInitialization) { *out = *in diff --git a/config/crd/bases/crds.yaml b/config/crd/bases/crds.yaml index 9a0d5b0d..0f7dfbee 100644 --- a/config/crd/bases/crds.yaml +++ b/config/crd/bases/crds.yaml @@ -1628,6 +1628,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -3500,6 +3516,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -6045,6 +6077,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -7917,6 +7965,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -9418,10 +9482,6 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - clusterId: - description: ClusterId is the identifier of computeCluster, - this will distinguish all computeCluster in meta. - type: string configMaps: description: ConfigMaps describe all configmap that need to be mounted. @@ -9788,12 +9848,6 @@ spec: description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object - name: - description: 'Name is the identifier of computeCluster, name - can be used specify what computeCluster to run sql. if not - set, will use `computeCluster` and the index in array to set.ep: - computeCluster-1.' - type: string noStoreLog: description: when set true, the log will store in disk that created by volumeClaimTemplate @@ -10368,6 +10422,12 @@ spec: type: string type: object type: array + uniqueId: + description: the unique identifier of compute cluster, first + register in fe will use UniqueId as cluster name. + type: string + required: + - uniqueId type: object type: array feSpec: @@ -14049,9 +14109,6 @@ spec: status. items: properties: - ComputeClusterName: - description: represents the compute cluster. - type: string availableReplicas: description: Total number of available pods (ready for at least minReadySeconds) targeted by this statefulset. @@ -14086,6 +14143,8 @@ spec: cluster before resume. format: int32 type: integer + uniqueId: + type: string type: object type: array feStatus: diff --git a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml index 688f76f3..cfdabf57 100644 --- a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml +++ b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml @@ -972,10 +972,6 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - clusterId: - description: ClusterId is the identifier of computeCluster, - this will distinguish all computeCluster in meta. - type: string configMaps: description: ConfigMaps describe all configmap that need to be mounted. @@ -1342,12 +1338,6 @@ spec: description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object - name: - description: 'Name is the identifier of computeCluster, name - can be used specify what computeCluster to run sql. if not - set, will use `computeCluster` and the index in array to set.ep: - computeCluster-1.' - type: string noStoreLog: description: when set true, the log will store in disk that created by volumeClaimTemplate @@ -1922,6 +1912,12 @@ spec: type: string type: object type: array + uniqueId: + description: the unique identifier of compute cluster, first + register in fe will use UniqueId as cluster name. + type: string + required: + - uniqueId type: object type: array feSpec: @@ -5603,9 +5599,6 @@ spec: status. items: properties: - ComputeClusterName: - description: represents the compute cluster. - type: string availableReplicas: description: Total number of available pods (ready for at least minReadySeconds) targeted by this statefulset. @@ -5640,6 +5633,8 @@ spec: cluster before resume. format: int32 type: integer + uniqueId: + type: string type: object type: array feStatus: diff --git a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml index ce91075d..c07c1c85 100644 --- a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml +++ b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml @@ -1628,6 +1628,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -3500,6 +3516,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -6045,6 +6077,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -7917,6 +7965,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: diff --git a/doc/examples/doriscluster-sample-secret.yaml b/doc/examples/doriscluster-sample-secret.yaml new file mode 100644 index 00000000..c3bf9d59 --- /dev/null +++ b/doc/examples/doriscluster-sample-secret.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# this yaml describe `secret` config in DorisCluster CRD. +#Compareed to `doriscluster-sample.yaml`, It have `secret` config for fe +apiVersion: doris.selectdb.com/v1 +kind: DorisCluster +metadata: + labels: + app.kubernetes.io/name: doriscluster + app.kubernetes.io/instance: doriscluster-sample-secret + app.kubernetes.io/part-of: doris-operator + name: doriscluster-sample-secret +spec: + feSpec: + replicas: 3 + image: selectdb/doris.fe-ubuntu:2.1.1 + limits: + cpu: 8 + memory: 16Gi + requests: + cpu: 8 + memory: 16Gi + secrets: + # use kubectl create secret generic db-user --from-file=./username.txt -n doris + - secretName: db-user + mountPath: /etc/doris + readOnly: true + # use kubectl create secret generic db-pass --from-file=./password.txt -n doris + - secretName: db-pass + mountPath: /opt/doris + readOnly: true + beSpec: + replicas: 3 + image: selectdb/doris.be-ubuntu:2.1.1 + limits: + cpu: 8 + memory: 16Gi + requests: + cpu: 8 + memory: 16Gi diff --git a/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml b/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml index ce91075d..c07c1c85 100644 --- a/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml +++ b/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml @@ -1628,6 +1628,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -3500,6 +3516,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -6045,6 +6077,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -7917,6 +7965,22 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: diff --git a/pkg/common/utils/k8s/client.go b/pkg/common/utils/k8s/client.go index 43d8f7e9..42384cd2 100644 --- a/pkg/common/utils/k8s/client.go +++ b/pkg/common/utils/k8s/client.go @@ -229,6 +229,20 @@ func GetConfigMap(ctx context.Context, k8scient client.Client, namespace, name s return &configMap, nil } +// CheckSecretExist check if the secret is in the namespace. +func CheckSecretExist(ctx context.Context, k8scient client.Client, namespace string, secrets []dorisv1.Secret) { + errMessage := "" + for _, secret := range secrets { + var s corev1.Secret + if getErr := k8scient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: secret.SecretName}, &s); getErr != nil { + errMessage = errMessage + fmt.Sprintf("(name: %s, namespace: %s, err: %s), ", secret.SecretName, namespace, getErr.Error()) + } + } + if errMessage != "" { + klog.Errorf("CheckSecretExist error: %s.", errMessage) + } +} + // GetConfigMaps get the configmap by the array of MountConfigMapInfo and namespace. func GetConfigMaps(ctx context.Context, k8scient client.Client, namespace string, cms []dorisv1.MountConfigMapInfo) ([]*corev1.ConfigMap, error) { var configMaps []*corev1.ConfigMap diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index a2c35780..17116d76 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -129,6 +129,11 @@ func NewPodTemplateSpec(dcr *v1.DorisCluster, componentType v1.ComponentType) co volumes = append(volumes, configVolumes...) } + if len(spec.Secrets) != 0 { + secretVolumes, _ := getMultiSecretVolumeAndVolumeMount(spec, componentType) + volumes = append(volumes, secretVolumes...) + } + pts := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: GeneratePodTemplateName(dcr, componentType), @@ -364,6 +369,11 @@ func NewBaseMainContainer(dcr *v1.DorisCluster, config map[string]interface{}, c volumeMounts = append(volumeMounts, configVolumeMounts...) } + if len(spec.Secrets) != 0 { + _, secretVolumeMounts := getMultiSecretVolumeAndVolumeMount(&spec, componentType) + volumeMounts = append(volumeMounts, secretVolumeMounts...) + } + // add basic auth secret volumeMount if dcr.Spec.AuthSecret != "" { volumeMounts = append(volumeMounts, corev1.VolumeMount{ @@ -678,6 +688,55 @@ func getMultiConfigVolumeAndVolumeMount(cmInfo *v1.ConfigMapInfo, componentType return volumes, volumeMounts } +func getMultiSecretVolumeAndVolumeMount(bSpec *v1.BaseSpec, componentType v1.ComponentType) ([]corev1.Volume, []corev1.VolumeMount) { + var volumes []corev1.Volume + var volumeMounts []corev1.VolumeMount + + if bSpec.Secrets == nil { + return volumes, volumeMounts + } + + defaultMountPath := "" + switch componentType { + case v1.Component_FE, v1.Component_BE, v1.Component_CN, v1.Component_Broker: + defaultMountPath = config_env_path + default: + klog.Infof("getMultiSecretVolumeAndVolumeMount componentType %s not supported.", componentType) + } + + for _, secret := range bSpec.Secrets { + path := secret.MountPath + if secret.MountPath == "" { + path = defaultMountPath + } + volumes = append( + volumes, + corev1.Volume{ + Name: secret.SecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.SecretName, + }, + }, + }, + ) + + readOnly := false + if secret.ReadOnly == true { + readOnly = secret.ReadOnly + } + volumeMounts = append( + volumeMounts, + corev1.VolumeMount{ + Name: secret.SecretName, + MountPath: path, + ReadOnly: readOnly, + }, + ) + } + return volumes, volumeMounts +} + func LivenessProbe(port int32, path string, commands []string, pt ProbeType) *corev1.Probe { return livenessProbe(port, path, commands, pt) } diff --git a/pkg/controller/sub_controller/be/controller.go b/pkg/controller/sub_controller/be/controller.go index f8eb62de..ee9d7748 100644 --- a/pkg/controller/sub_controller/be/controller.go +++ b/pkg/controller/sub_controller/be/controller.go @@ -67,7 +67,9 @@ func (be *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { return err } + k8s.CheckSecretExist(ctx, be.K8sclient, dcr.Namespace, dcr.Spec.BeSpec.Secrets) be.CheckConfigMountPath(dcr, v1.Component_BE) + be.CheckSecretMountPath(dcr, v1.Component_BE) //generate new be service. svc := resource.BuildExternalService(dcr, v1.Component_BE, config) //create or update be external and domain search service, update the status of fe on src. diff --git a/pkg/controller/sub_controller/broker/controller.go b/pkg/controller/sub_controller/broker/controller.go index 0fa672b1..87fcf575 100644 --- a/pkg/controller/sub_controller/broker/controller.go +++ b/pkg/controller/sub_controller/broker/controller.go @@ -71,7 +71,9 @@ func (bk *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { klog.Error("BrokerController Sync ", "resolve broker configmap failed, namespace ", dcr.Namespace, " error ", err) return err } + k8s.CheckSecretExist(ctx, bk.K8sclient, dcr.Namespace, dcr.Spec.BrokerSpec.Secrets) bk.CheckConfigMountPath(dcr, v1.Component_Broker) + bk.CheckSecretMountPath(dcr, v1.Component_Broker) internalService := resource.BuildInternalService(dcr, v1.Component_Broker, config) if err := k8s.ApplyService(ctx, bk.K8sclient, &internalService, resource.ServiceDeepEqual); err != nil { klog.Errorf("broker controller sync apply internalService name=%s, namespace=%s, clusterName=%s failed.message=%s.", diff --git a/pkg/controller/sub_controller/cn/controller.go b/pkg/controller/sub_controller/cn/controller.go index cf4951cc..8a3b97ce 100644 --- a/pkg/controller/sub_controller/cn/controller.go +++ b/pkg/controller/sub_controller/cn/controller.go @@ -74,7 +74,9 @@ func (cn *Controller) Sync(ctx context.Context, dcr *dorisv1.DorisCluster) error klog.Errorf("cn controller sync resolve cn configMap failed, namespace %s ，err :", dcr.Namespace, err) return err } + k8s.CheckSecretExist(ctx, cn.K8sclient, dcr.Namespace, dcr.Spec.CnSpec.Secrets) cn.CheckConfigMountPath(dcr, dorisv1.Component_CN) + cn.CheckSecretMountPath(dcr, dorisv1.Component_CN) svc := resource.BuildExternalService(dcr, dorisv1.Component_CN, config) internalSVC := resource.BuildInternalService(dcr, dorisv1.Component_CN, config) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computeclusters/controller.go b/pkg/controller/sub_controller/disaggregated_cluster/computeclusters/controller.go index bbd3a260..8c461adf 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computeclusters/controller.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computeclusters/controller.go @@ -257,7 +257,7 @@ func (dccs *DisaggregatedComputeClustersController) initialCCStatus(ddc *dv1.Dor defaultStatus := dv1.ComputeClusterStatus{ Phase: dv1.Reconciling, ClusterId: clusterId, - UniqueId: cc.UniqueId, + UniqueId: cc.UniqueId, StatefulsetName: ddc.GetCCStatefulsetName(cc), ServiceName: ddc.GetCCServiceName(cc), //set for status updated. diff --git a/pkg/controller/sub_controller/events.go b/pkg/controller/sub_controller/events.go index 61f5de4c..1ff216cb 100644 --- a/pkg/controller/sub_controller/events.go +++ b/pkg/controller/sub_controller/events.go @@ -71,6 +71,7 @@ var ( InstanceMetaCreated EventReason = "InstanceMetaCreated" InstanceIdModified EventReason = "InstanceIdModified" ConfigMapPathRepeated EventReason = "ConfigMapPathRepeated" + SecretPathRepeated EventReason = "SecretPathRepeated" WaitMetaServiceAvailable EventReason = "WaitMetaServiceAvailable" WaitFEAvailable EventReason = "WaitFEAvailable" ServiceApplyedFailed EventReason = "ServiceApplyedFailed" diff --git a/pkg/controller/sub_controller/fe/controller.go b/pkg/controller/sub_controller/fe/controller.go index 75b6d42d..6b1cc09d 100644 --- a/pkg/controller/sub_controller/fe/controller.go +++ b/pkg/controller/sub_controller/fe/controller.go @@ -89,7 +89,9 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error klog.Error("fe Controller Sync ", "resolve fe configmap failed, namespace ", cluster.Namespace, " error :", err) return err } + k8s.CheckSecretExist(ctx, fc.K8sclient, cluster.Namespace, cluster.Spec.FeSpec.Secrets) fc.CheckConfigMountPath(cluster, v1.Component_FE) + fc.CheckSecretMountPath(cluster, v1.Component_FE) //generate new fe service. svc := resource.BuildExternalService(cluster, v1.Component_FE, config) diff --git a/pkg/controller/sub_controller/sub_controller.go b/pkg/controller/sub_controller/sub_controller.go index d443e5b1..32fa6f5b 100644 --- a/pkg/controller/sub_controller/sub_controller.go +++ b/pkg/controller/sub_controller/sub_controller.go @@ -125,6 +125,9 @@ func (d *SubDefaultController) CheckConfigMountPath(dcr *dorisv1.DorisCluster, c var mountsMap = make(map[string]dorisv1.MountConfigMapInfo) for _, cm := range cms { path := cm.MountPath + if cm.MountPath == "" { + path = resource.ConfigEnvPath + "[default]" + } if m, exist := mountsMap[path]; exist { klog.Errorf("CheckConfigMountPath error: the mountPath %s is repeated between configmap: %s and configmap: %s.", path, cm.ConfigMapName, m.ConfigMapName) d.K8srecorder.Event(dcr, string(EventWarning), string(ConfigMapPathRepeated), fmt.Sprintf("the mountPath %s is repeated between configmap: %s and configmap: %s.", path, cm.ConfigMapName, m.ConfigMapName)) @@ -133,6 +136,35 @@ func (d *SubDefaultController) CheckConfigMountPath(dcr *dorisv1.DorisCluster, c } } +// generate map for mountpath:secret +func (d *SubDefaultController) CheckSecretMountPath(dcr *dorisv1.DorisCluster, componentType dorisv1.ComponentType) { + var secrets []dorisv1.Secret + switch componentType { + case dorisv1.Component_FE: + secrets = dcr.Spec.FeSpec.Secrets + case dorisv1.Component_BE: + secrets = dcr.Spec.BeSpec.Secrets + case dorisv1.Component_CN: + secrets = dcr.Spec.CnSpec.Secrets + case dorisv1.Component_Broker: + secrets = dcr.Spec.BrokerSpec.Secrets + default: + klog.Infof("the componentType %s is not supported.", componentType) + } + var mountsMap = make(map[string]dorisv1.Secret) + for _, secret := range secrets { + path := secret.MountPath + if secret.MountPath == "" { + path = resource.ConfigEnvPath + "[default]" + } + if s, exist := mountsMap[path]; exist { + klog.Errorf("CheckSecretMountPath error: the mountPath %s is repeated between secret: %s and secret: %s.", path, secret.SecretName, s.SecretName) + d.K8srecorder.Event(dcr, string(EventWarning), string(SecretPathRepeated), fmt.Sprintf("the mountPath %s is repeated between secret: %s and secret: %s.", path, secret.SecretName, s.SecretName)) + } + mountsMap[path] = secret + } +} + // ClearCommonResources clear common resources all component have, as statefulset, service. // response `bool` represents all resource have deleted, if not and delete resource failed return false for next reconcile retry. func (d *SubDefaultController) ClearCommonResources(ctx context.Context, dcr *dorisv1.DorisCluster, componentType dorisv1.ComponentType) (bool, error) { From f8c4cba56b7f529ba960113ecbbcc2670fb37cc9 Mon Sep 17 00:00:00 2001 From: hechao Date: Wed, 18 Sep 2024 15:06:27 +0800 Subject: [PATCH 2/4] multi secret --- api/doris/v1/types.go | 6 +---- config/crd/bases/crds.yaml | 24 +++++++------------ .../doris.selectdb.com_dorisclusters.yaml | 24 +++++++------------ .../doris.selectdb.com_dorisclusters.yaml | 24 +++++++------------ pkg/common/utils/resource/pod.go | 5 ---- .../sub_controller/be/controller.go | 6 ++++- .../sub_controller/broker/controller.go | 6 ++++- .../sub_controller/cn/controller.go | 6 ++++- .../sub_controller/fe/controller.go | 6 ++++- .../sub_controller/sub_controller.go | 6 ----- 10 files changed, 45 insertions(+), 68 deletions(-) diff --git a/api/doris/v1/types.go b/api/doris/v1/types.go index 90536eb8..495da02c 100644 --- a/api/doris/v1/types.go +++ b/api/doris/v1/types.go @@ -290,13 +290,9 @@ type Secret struct { SecretName string `json:"secretName,omitempty"` // Current Secret Mount Path, default is "/etc/doris" + // If Secret belongs to the same Secrets, their MountPath cannot be repeated. // +optional MountPath string `json:"mountPath,omitempty"` - - // Mounted read-only if true, read-write otherwise (false or unspecified). - // Defaults to false. - // +optional - ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,2,opt,name=readOnly"` } // ExportService consisting of expose ports for user access to software service. diff --git a/config/crd/bases/crds.yaml b/config/crd/bases/crds.yaml index f20cfdf2..90b9472c 100644 --- a/config/crd/bases/crds.yaml +++ b/config/crd/bases/crds.yaml @@ -1634,11 +1634,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -3522,11 +3520,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -6083,11 +6079,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -7971,11 +7965,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string diff --git a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml index c07c1c85..e40b2070 100644 --- a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml +++ b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml @@ -1634,11 +1634,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -3522,11 +3520,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -6083,11 +6079,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -7971,11 +7965,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string diff --git a/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml b/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml index c07c1c85..e40b2070 100644 --- a/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml +++ b/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml @@ -1634,11 +1634,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -3522,11 +3520,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -6083,11 +6079,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string @@ -7971,11 +7965,9 @@ spec: properties: mountPath: description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + cannot be repeated. type: string - readOnly: - description: Mounted read-only if true, read-write otherwise - (false or unspecified). Defaults to false. - type: boolean secretName: description: name of secret that needs to mount. type: string diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index 17116d76..663b9ac0 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -721,16 +721,11 @@ func getMultiSecretVolumeAndVolumeMount(bSpec *v1.BaseSpec, componentType v1.Com }, ) - readOnly := false - if secret.ReadOnly == true { - readOnly = secret.ReadOnly - } volumeMounts = append( volumeMounts, corev1.VolumeMount{ Name: secret.SecretName, MountPath: path, - ReadOnly: readOnly, }, ) } diff --git a/pkg/controller/sub_controller/be/controller.go b/pkg/controller/sub_controller/be/controller.go index ee9d7748..155d8efa 100644 --- a/pkg/controller/sub_controller/be/controller.go +++ b/pkg/controller/sub_controller/be/controller.go @@ -67,7 +67,6 @@ func (be *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { return err } - k8s.CheckSecretExist(ctx, be.K8sclient, dcr.Namespace, dcr.Spec.BeSpec.Secrets) be.CheckConfigMountPath(dcr, v1.Component_BE) be.CheckSecretMountPath(dcr, v1.Component_BE) //generate new be service. @@ -101,6 +100,11 @@ func (be *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { return err } + // check if the secret exists in the namespace + if dcr.Spec.BeSpec.Secrets != nil { + k8s.CheckSecretExist(ctx, be.K8sclient, dcr.Namespace, dcr.Spec.BeSpec.Secrets) + } + return nil } diff --git a/pkg/controller/sub_controller/broker/controller.go b/pkg/controller/sub_controller/broker/controller.go index 87fcf575..d1c46474 100644 --- a/pkg/controller/sub_controller/broker/controller.go +++ b/pkg/controller/sub_controller/broker/controller.go @@ -71,7 +71,6 @@ func (bk *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { klog.Error("BrokerController Sync ", "resolve broker configmap failed, namespace ", dcr.Namespace, " error ", err) return err } - k8s.CheckSecretExist(ctx, bk.K8sclient, dcr.Namespace, dcr.Spec.BrokerSpec.Secrets) bk.CheckConfigMountPath(dcr, v1.Component_Broker) bk.CheckSecretMountPath(dcr, v1.Component_Broker) internalService := resource.BuildInternalService(dcr, v1.Component_Broker, config) @@ -91,6 +90,11 @@ func (bk *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { return err } + // check if the secret exists in the namespace + if dcr.Spec.BrokerSpec.Secrets != nil { + k8s.CheckSecretExist(ctx, bk.K8sclient, dcr.Namespace, dcr.Spec.BrokerSpec.Secrets) + } + return nil } diff --git a/pkg/controller/sub_controller/cn/controller.go b/pkg/controller/sub_controller/cn/controller.go index 8a3b97ce..2662706b 100644 --- a/pkg/controller/sub_controller/cn/controller.go +++ b/pkg/controller/sub_controller/cn/controller.go @@ -74,7 +74,6 @@ func (cn *Controller) Sync(ctx context.Context, dcr *dorisv1.DorisCluster) error klog.Errorf("cn controller sync resolve cn configMap failed, namespace %s ，err :", dcr.Namespace, err) return err } - k8s.CheckSecretExist(ctx, cn.K8sclient, dcr.Namespace, dcr.Spec.CnSpec.Secrets) cn.CheckConfigMountPath(dcr, dorisv1.Component_CN) cn.CheckSecretMountPath(dcr, dorisv1.Component_CN) svc := resource.BuildExternalService(dcr, dorisv1.Component_CN, config) @@ -103,6 +102,11 @@ func (cn *Controller) Sync(ctx context.Context, dcr *dorisv1.DorisCluster) error return err } + // check if the secret exists in the namespace + if dcr.Spec.CnSpec.Secrets != nil { + k8s.CheckSecretExist(ctx, cn.K8sclient, dcr.Namespace, dcr.Spec.CnSpec.Secrets) + } + //create autoscaler. if cnSpec.AutoScalingPolicy != nil { err = cn.deployAutoScaler(ctx, *cnSpec.AutoScalingPolicy, &cnStatefulSet, dcr) diff --git a/pkg/controller/sub_controller/fe/controller.go b/pkg/controller/sub_controller/fe/controller.go index 6b1cc09d..9cfbb737 100644 --- a/pkg/controller/sub_controller/fe/controller.go +++ b/pkg/controller/sub_controller/fe/controller.go @@ -89,7 +89,6 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error klog.Error("fe Controller Sync ", "resolve fe configmap failed, namespace ", cluster.Namespace, " error :", err) return err } - k8s.CheckSecretExist(ctx, fc.K8sclient, cluster.Namespace, cluster.Spec.FeSpec.Secrets) fc.CheckConfigMountPath(cluster, v1.Component_FE) fc.CheckSecretMountPath(cluster, v1.Component_FE) @@ -127,5 +126,10 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error return err } + // check if the secret exists in the namespace + if cluster.Spec.FeSpec.Secrets != nil { + k8s.CheckSecretExist(ctx, fc.K8sclient, cluster.Namespace, cluster.Spec.FeSpec.Secrets) + } + return nil } diff --git a/pkg/controller/sub_controller/sub_controller.go b/pkg/controller/sub_controller/sub_controller.go index 32fa6f5b..46bb46b9 100644 --- a/pkg/controller/sub_controller/sub_controller.go +++ b/pkg/controller/sub_controller/sub_controller.go @@ -125,9 +125,6 @@ func (d *SubDefaultController) CheckConfigMountPath(dcr *dorisv1.DorisCluster, c var mountsMap = make(map[string]dorisv1.MountConfigMapInfo) for _, cm := range cms { path := cm.MountPath - if cm.MountPath == "" { - path = resource.ConfigEnvPath + "[default]" - } if m, exist := mountsMap[path]; exist { klog.Errorf("CheckConfigMountPath error: the mountPath %s is repeated between configmap: %s and configmap: %s.", path, cm.ConfigMapName, m.ConfigMapName) d.K8srecorder.Event(dcr, string(EventWarning), string(ConfigMapPathRepeated), fmt.Sprintf("the mountPath %s is repeated between configmap: %s and configmap: %s.", path, cm.ConfigMapName, m.ConfigMapName)) @@ -154,9 +151,6 @@ func (d *SubDefaultController) CheckSecretMountPath(dcr *dorisv1.DorisCluster, c var mountsMap = make(map[string]dorisv1.Secret) for _, secret := range secrets { path := secret.MountPath - if secret.MountPath == "" { - path = resource.ConfigEnvPath + "[default]" - } if s, exist := mountsMap[path]; exist { klog.Errorf("CheckSecretMountPath error: the mountPath %s is repeated between secret: %s and secret: %s.", path, secret.SecretName, s.SecretName) d.K8srecorder.Event(dcr, string(EventWarning), string(SecretPathRepeated), fmt.Sprintf("the mountPath %s is repeated between secret: %s and secret: %s.", path, secret.SecretName, s.SecretName)) From 8ffbf552242f199425357349e2cf026bb49375e8 Mon Sep 17 00:00:00 2001 From: hechao Date: Wed, 18 Sep 2024 15:14:41 +0800 Subject: [PATCH 3/4] multi secret --- doc/examples/doriscluster-sample-secret.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/examples/doriscluster-sample-secret.yaml b/doc/examples/doriscluster-sample-secret.yaml index c3bf9d59..83dee278 100644 --- a/doc/examples/doriscluster-sample-secret.yaml +++ b/doc/examples/doriscluster-sample-secret.yaml @@ -39,11 +39,9 @@ spec: # use kubectl create secret generic db-user --from-file=./username.txt -n doris - secretName: db-user mountPath: /etc/doris - readOnly: true # use kubectl create secret generic db-pass --from-file=./password.txt -n doris - secretName: db-pass mountPath: /opt/doris - readOnly: true beSpec: replicas: 3 image: selectdb/doris.be-ubuntu:2.1.1 From b08c22ef7c9f708f7f5188d9e53719632b9f3335 Mon Sep 17 00:00:00 2001 From: hechao Date: Mon, 23 Sep 2024 18:15:22 +0800 Subject: [PATCH 4/4] multi secret --- api/doris/v1/types.go | 2 +- config/crd/bases/crds.yaml | 8 +++--- .../doris.selectdb.com_dorisclusters.yaml | 8 +++--- doc/examples/doriscluster-sample-secret.yaml | 18 ++++++++++-- .../doris.selectdb.com_dorisclusters.yaml | 8 +++--- pkg/common/utils/k8s/client.go | 14 ---------- pkg/common/utils/resource/pod.go | 25 ++++++++--------- .../sub_controller/be/controller.go | 6 +--- .../sub_controller/broker/controller.go | 6 +--- .../sub_controller/cn/controller.go | 6 +--- pkg/controller/sub_controller/events.go | 1 + .../sub_controller/fe/controller.go | 6 +--- .../sub_controller/sub_controller.go | 28 +++++++++++++++++++ 13 files changed, 73 insertions(+), 63 deletions(-) diff --git a/api/doris/v1/types.go b/api/doris/v1/types.go index 495da02c..4d9e1d09 100644 --- a/api/doris/v1/types.go +++ b/api/doris/v1/types.go @@ -290,7 +290,7 @@ type Secret struct { SecretName string `json:"secretName,omitempty"` // Current Secret Mount Path, default is "/etc/doris" - // If Secret belongs to the same Secrets, their MountPath cannot be repeated. + // If Secret belongs to the same Secrets, their mountPath can't be repeated. // +optional MountPath string `json:"mountPath,omitempty"` } diff --git a/config/crd/bases/crds.yaml b/config/crd/bases/crds.yaml index 90b9472c..fe07a973 100644 --- a/config/crd/bases/crds.yaml +++ b/config/crd/bases/crds.yaml @@ -1635,7 +1635,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -3521,7 +3521,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -6080,7 +6080,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -7966,7 +7966,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. diff --git a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml index e40b2070..2a764dcf 100644 --- a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml +++ b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml @@ -1635,7 +1635,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -3521,7 +3521,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -6080,7 +6080,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -7966,7 +7966,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. diff --git a/doc/examples/doriscluster-sample-secret.yaml b/doc/examples/doriscluster-sample-secret.yaml index 83dee278..e4305376 100644 --- a/doc/examples/doriscluster-sample-secret.yaml +++ b/doc/examples/doriscluster-sample-secret.yaml @@ -17,6 +17,22 @@ # this yaml describe `secret` config in DorisCluster CRD. #Compareed to `doriscluster-sample.yaml`, It have `secret` config for fe +apiVersion: v1 +kind: Secret +metadata: + name: db-user +type: Opaque +data: + user: cm9vdAo= +--- +apiVersion: v1 +kind: Secret +metadata: + name: db-pass +type: Opaque +data: + password: MTIzNDU2Cg== +--- apiVersion: doris.selectdb.com/v1 kind: DorisCluster metadata: @@ -36,10 +52,8 @@ spec: cpu: 8 memory: 16Gi secrets: - # use kubectl create secret generic db-user --from-file=./username.txt -n doris - secretName: db-user mountPath: /etc/doris - # use kubectl create secret generic db-pass --from-file=./password.txt -n doris - secretName: db-pass mountPath: /opt/doris beSpec: diff --git a/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml b/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml index e40b2070..2a764dcf 100644 --- a/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml +++ b/helm-charts/doris-operator/crds/doris.selectdb.com_dorisclusters.yaml @@ -1635,7 +1635,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -3521,7 +3521,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -6080,7 +6080,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. @@ -7966,7 +7966,7 @@ spec: mountPath: description: Current Secret Mount Path, default is "/etc/doris" If Secret belongs to the same Secrets, their MountPath - cannot be repeated. + can't be repeated. type: string secretName: description: name of secret that needs to mount. diff --git a/pkg/common/utils/k8s/client.go b/pkg/common/utils/k8s/client.go index 870e776b..5a9129d2 100644 --- a/pkg/common/utils/k8s/client.go +++ b/pkg/common/utils/k8s/client.go @@ -229,20 +229,6 @@ func GetConfigMap(ctx context.Context, k8scient client.Client, namespace, name s return &configMap, nil } -// CheckSecretExist check if the secret is in the namespace. -func CheckSecretExist(ctx context.Context, k8scient client.Client, namespace string, secrets []dorisv1.Secret) { - errMessage := "" - for _, secret := range secrets { - var s corev1.Secret - if getErr := k8scient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: secret.SecretName}, &s); getErr != nil { - errMessage = errMessage + fmt.Sprintf("(name: %s, namespace: %s, err: %s), ", secret.SecretName, namespace, getErr.Error()) - } - } - if errMessage != "" { - klog.Errorf("CheckSecretExist error: %s.", errMessage) - } -} - // GetConfigMaps get the configmap by the array of MountConfigMapInfo and namespace. func GetConfigMaps(ctx context.Context, k8scient client.Client, namespace string, cms []dorisv1.MountConfigMapInfo) ([]*corev1.ConfigMap, error) { var configMaps []*corev1.ConfigMap diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index a86a149e..c9980492 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -29,15 +29,16 @@ import ( ) const ( - config_env_path = "/etc/doris" - ConfigEnvPath = config_env_path - config_env_name = "CONFIGMAP_MOUNT_PATH" - basic_auth_path = "/etc/basic_auth" - auth_volume_name = "basic-auth" - be_storage_name = "be-storage" - be_storage_path = "/opt/apache-doris/be/storage" - fe_meta_path = "/opt/apache-doris/fe/doris-meta" - fe_meta_name = "fe-meta" + config_env_path = "/etc/doris" + ConfigEnvPath = config_env_path + secret_config_path = config_env_path + config_env_name = "CONFIGMAP_MOUNT_PATH" + basic_auth_path = "/etc/basic_auth" + auth_volume_name = "basic-auth" + be_storage_name = "be-storage" + be_storage_path = "/opt/apache-doris/be/storage" + fe_meta_path = "/opt/apache-doris/fe/doris-meta" + fe_meta_name = "fe-meta" HEALTH_API_PATH = "/api/health" HEALTH_BROKER_LIVE_COMMAND = "/opt/apache-doris/broker_is_alive.sh" @@ -692,14 +693,10 @@ func getMultiSecretVolumeAndVolumeMount(bSpec *v1.BaseSpec, componentType v1.Com var volumes []corev1.Volume var volumeMounts []corev1.VolumeMount - if bSpec.Secrets == nil { - return volumes, volumeMounts - } - defaultMountPath := "" switch componentType { case v1.Component_FE, v1.Component_BE, v1.Component_CN, v1.Component_Broker: - defaultMountPath = config_env_path + defaultMountPath = secret_config_path default: klog.Infof("getMultiSecretVolumeAndVolumeMount componentType %s not supported.", componentType) } diff --git a/pkg/controller/sub_controller/be/controller.go b/pkg/controller/sub_controller/be/controller.go index a5f47525..040027f2 100644 --- a/pkg/controller/sub_controller/be/controller.go +++ b/pkg/controller/sub_controller/be/controller.go @@ -69,6 +69,7 @@ func (be *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { be.CheckConfigMountPath(dcr, v1.Component_BE) be.CheckSecretMountPath(dcr, v1.Component_BE) + be.CheckSecretExist(ctx, dcr, v1.Component_BE) //generate new be service. svc := resource.BuildExternalService(dcr, v1.Component_BE, config) //create or update be external and domain search service, update the status of fe on src. @@ -100,11 +101,6 @@ func (be *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { return err } - // check if the secret exists in the namespace - if dcr.Spec.BeSpec.Secrets != nil { - k8s.CheckSecretExist(ctx, be.K8sclient, dcr.Namespace, dcr.Spec.BeSpec.Secrets) - } - return nil } diff --git a/pkg/controller/sub_controller/broker/controller.go b/pkg/controller/sub_controller/broker/controller.go index ef115a37..e8a67154 100644 --- a/pkg/controller/sub_controller/broker/controller.go +++ b/pkg/controller/sub_controller/broker/controller.go @@ -73,6 +73,7 @@ func (bk *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { } bk.CheckConfigMountPath(dcr, v1.Component_Broker) bk.CheckSecretMountPath(dcr, v1.Component_Broker) + bk.CheckSecretExist(ctx, dcr, v1.Component_Broker) internalService := resource.BuildInternalService(dcr, v1.Component_Broker, config) if err := k8s.ApplyService(ctx, bk.K8sclient, &internalService, resource.ServiceDeepEqual); err != nil { klog.Errorf("broker controller sync apply internalService name=%s, namespace=%s, clusterName=%s failed.message=%s.", @@ -90,11 +91,6 @@ func (bk *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { return err } - // check if the secret exists in the namespace - if dcr.Spec.BrokerSpec.Secrets != nil { - k8s.CheckSecretExist(ctx, bk.K8sclient, dcr.Namespace, dcr.Spec.BrokerSpec.Secrets) - } - return nil } diff --git a/pkg/controller/sub_controller/cn/controller.go b/pkg/controller/sub_controller/cn/controller.go index 9c8cbea6..6bb408c2 100644 --- a/pkg/controller/sub_controller/cn/controller.go +++ b/pkg/controller/sub_controller/cn/controller.go @@ -76,6 +76,7 @@ func (cn *Controller) Sync(ctx context.Context, dcr *dorisv1.DorisCluster) error } cn.CheckConfigMountPath(dcr, dorisv1.Component_CN) cn.CheckSecretMountPath(dcr, dorisv1.Component_CN) + cn.CheckSecretExist(ctx, dcr, dorisv1.Component_CN) svc := resource.BuildExternalService(dcr, dorisv1.Component_CN, config) internalSVC := resource.BuildInternalService(dcr, dorisv1.Component_CN, config) @@ -102,11 +103,6 @@ func (cn *Controller) Sync(ctx context.Context, dcr *dorisv1.DorisCluster) error return err } - // check if the secret exists in the namespace - if dcr.Spec.CnSpec.Secrets != nil { - k8s.CheckSecretExist(ctx, cn.K8sclient, dcr.Namespace, dcr.Spec.CnSpec.Secrets) - } - //create autoscaler. if cnSpec.AutoScalingPolicy != nil { err = cn.deployAutoScaler(ctx, *cnSpec.AutoScalingPolicy, &cnStatefulSet, dcr) diff --git a/pkg/controller/sub_controller/events.go b/pkg/controller/sub_controller/events.go index 5750fbb4..1ce69590 100644 --- a/pkg/controller/sub_controller/events.go +++ b/pkg/controller/sub_controller/events.go @@ -72,6 +72,7 @@ var ( InstanceIdModified EventReason = "InstanceIdModified" ConfigMapPathRepeated EventReason = "ConfigMapPathRepeated" SecretPathRepeated EventReason = "SecretPathRepeated" + SecretNotExist EventReason = "SecretNotExist" WaitMetaServiceAvailable EventReason = "WaitMetaServiceAvailable" WaitFEAvailable EventReason = "WaitFEAvailable" ServiceApplyedFailed EventReason = "ServiceApplyedFailed" diff --git a/pkg/controller/sub_controller/fe/controller.go b/pkg/controller/sub_controller/fe/controller.go index 3e4d1a21..936151cb 100644 --- a/pkg/controller/sub_controller/fe/controller.go +++ b/pkg/controller/sub_controller/fe/controller.go @@ -91,6 +91,7 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error } fc.CheckConfigMountPath(cluster, v1.Component_FE) fc.CheckSecretMountPath(cluster, v1.Component_FE) + fc.CheckSecretExist(ctx, cluster, v1.Component_FE) //generate new fe service. svc := resource.BuildExternalService(cluster, v1.Component_FE, config) @@ -126,10 +127,5 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error return err } - // check if the secret exists in the namespace - if cluster.Spec.FeSpec.Secrets != nil { - k8s.CheckSecretExist(ctx, fc.K8sclient, cluster.Namespace, cluster.Spec.FeSpec.Secrets) - } - return nil } diff --git a/pkg/controller/sub_controller/sub_controller.go b/pkg/controller/sub_controller/sub_controller.go index fa267814..e2c984b8 100644 --- a/pkg/controller/sub_controller/sub_controller.go +++ b/pkg/controller/sub_controller/sub_controller.go @@ -159,6 +159,34 @@ func (d *SubDefaultController) CheckSecretMountPath(dcr *dorisv1.DorisCluster, c } } +// CheckSecretExist, check the secret exist or not in specify namespace. +func (d *SubDefaultController) CheckSecretExist(ctx context.Context, dcr *dorisv1.DorisCluster, componentType dorisv1.ComponentType) { + var secrets []dorisv1.Secret + switch componentType { + case dorisv1.Component_FE: + secrets = dcr.Spec.FeSpec.Secrets + case dorisv1.Component_BE: + secrets = dcr.Spec.BeSpec.Secrets + case dorisv1.Component_CN: + secrets = dcr.Spec.CnSpec.Secrets + case dorisv1.Component_Broker: + secrets = dcr.Spec.BrokerSpec.Secrets + default: + klog.Infof("the componentType %s is not supported.", componentType) + } + errMessage := "" + for _, secret := range secrets { + var s corev1.Secret + if getErr := d.K8sclient.Get(ctx, types.NamespacedName{Namespace: dcr.Namespace, Name: secret.SecretName}, &s); getErr != nil { + errMessage = errMessage + fmt.Sprintf("(name: %s, namespace: %s, err: %s), ", secret.SecretName, dcr.Namespace, getErr.Error()) + } + } + if errMessage != "" { + klog.Errorf("CheckSecretExist error: %s.", errMessage) + d.K8srecorder.Event(dcr, string(EventWarning), string(SecretNotExist), fmt.Sprintf("CheckSecretExist error: %s.", errMessage)) + } +} + // ClearCommonResources clear common resources all component have, as statefulset, service. // response `bool` represents all resource have deleted, if not and delete resource failed return false for next reconcile retry. func (d *SubDefaultController) ClearCommonResources(ctx context.Context, dcr *dorisv1.DorisCluster, componentType dorisv1.ComponentType) (bool, error) {