diff --git a/api/doris/v1/types.go b/api/doris/v1/types.go index 7dc18ea..f07c383 100644 --- a/api/doris/v1/types.go +++ b/api/doris/v1/types.go @@ -218,6 +218,10 @@ type BaseSpec struct { //Security context for all containers running in the pod (unless they override it). //+optional ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` + + // Multi Secret for pod. + // +optional + Secrets []Secret `json:"secrets,omitempty"` } type SystemInitialization struct { @@ -291,6 +295,16 @@ type MountConfigMapInfo struct { MountPath string `json:"mountPath,omitempty"` } +type Secret struct { + // name of secret that needs to mount. + SecretName string `json:"secretName,omitempty"` + + // Current Secret Mount Path, default is "/etc/doris" + // If Secret belongs to the same Secrets, their mountPath can't be repeated. + // +optional + MountPath string `json:"mountPath,omitempty"` +} + // ExportService consisting of expose ports for user access to software service. type ExportService struct { //type of service,the possible value for the service type are : ClusterIP, NodePort, LoadBalancer,ExternalName. diff --git a/api/doris/v1/zz_generated.deepcopy.go b/api/doris/v1/zz_generated.deepcopy.go index dfbbdf8..75e3c74 100644 --- a/api/doris/v1/zz_generated.deepcopy.go +++ b/api/doris/v1/zz_generated.deepcopy.go @@ -160,6 +160,11 @@ func (in *BaseSpec) DeepCopyInto(out *BaseSpec) { *out = new(corev1.SecurityContext) (*in).DeepCopyInto(*out) } + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]Secret, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BaseSpec. @@ -885,6 +890,21 @@ func (in *ResourceMetricSource) DeepCopy() *ResourceMetricSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Secret) DeepCopyInto(out *Secret) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Secret. +func (in *Secret) DeepCopy() *Secret { + if in == nil { + return nil + } + out := new(Secret) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SystemInitialization) DeepCopyInto(out *SystemInitialization) { *out = *in diff --git a/config/crd/bases/crds.yaml b/config/crd/bases/crds.yaml index 4cd2c6e..c5c9582 100644 --- a/config/crd/bases/crds.yaml +++ b/config/crd/bases/crds.yaml @@ -1557,6 +1557,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -3344,6 +3358,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -5748,6 +5776,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -7535,6 +7577,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: diff --git a/config/crd/bases/doris.apache.com_dorisclusters.yaml b/config/crd/bases/doris.apache.com_dorisclusters.yaml index d7a1740..915eb7e 100644 --- a/config/crd/bases/doris.apache.com_dorisclusters.yaml +++ b/config/crd/bases/doris.apache.com_dorisclusters.yaml @@ -1557,6 +1557,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -3344,6 +3358,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -5748,6 +5776,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -7535,6 +7577,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: diff --git a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml index d7a1740..915eb7e 100644 --- a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml +++ b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml @@ -1557,6 +1557,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -3344,6 +3358,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -5748,6 +5776,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: @@ -7535,6 +7577,20 @@ spec: otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object + secrets: + description: Multi Secret for pod. + items: + properties: + mountPath: + description: Current Secret Mount Path, default is "/etc/doris" + If Secret belongs to the same Secrets, their MountPath + can't be repeated. + type: string + secretName: + description: name of secret that needs to mount. + type: string + type: object + type: array securityContext: description: Security context for pod. properties: diff --git a/doc/examples/doriscluster-sample-secret.yaml b/doc/examples/doriscluster-sample-secret.yaml new file mode 100644 index 0000000..e430537 --- /dev/null +++ b/doc/examples/doriscluster-sample-secret.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# this yaml describe `secret` config in DorisCluster CRD. +#Compareed to `doriscluster-sample.yaml`, It have `secret` config for fe +apiVersion: v1 +kind: Secret +metadata: + name: db-user +type: Opaque +data: + user: cm9vdAo= +--- +apiVersion: v1 +kind: Secret +metadata: + name: db-pass +type: Opaque +data: + password: MTIzNDU2Cg== +--- +apiVersion: doris.selectdb.com/v1 +kind: DorisCluster +metadata: + labels: + app.kubernetes.io/name: doriscluster + app.kubernetes.io/instance: doriscluster-sample-secret + app.kubernetes.io/part-of: doris-operator + name: doriscluster-sample-secret +spec: + feSpec: + replicas: 3 + image: selectdb/doris.fe-ubuntu:2.1.1 + limits: + cpu: 8 + memory: 16Gi + requests: + cpu: 8 + memory: 16Gi + secrets: + - secretName: db-user + mountPath: /etc/doris + - secretName: db-pass + mountPath: /opt/doris + beSpec: + replicas: 3 + image: selectdb/doris.be-ubuntu:2.1.1 + limits: + cpu: 8 + memory: 16Gi + requests: + cpu: 8 + memory: 16Gi diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index caca70c..ea6388a 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -29,15 +29,16 @@ import ( ) const ( - config_env_path = "/etc/doris" - ConfigEnvPath = config_env_path - config_env_name = "CONFIGMAP_MOUNT_PATH" - basic_auth_path = "/etc/basic_auth" - auth_volume_name = "basic-auth" - be_storage_name = "be-storage" - be_storage_path = "/opt/apache-doris/be/storage" - fe_meta_path = "/opt/apache-doris/fe/doris-meta" - fe_meta_name = "fe-meta" + config_env_path = "/etc/doris" + ConfigEnvPath = config_env_path + secret_config_path = config_env_path + config_env_name = "CONFIGMAP_MOUNT_PATH" + basic_auth_path = "/etc/basic_auth" + auth_volume_name = "basic-auth" + be_storage_name = "be-storage" + be_storage_path = "/opt/apache-doris/be/storage" + fe_meta_path = "/opt/apache-doris/fe/doris-meta" + fe_meta_name = "fe-meta" HEALTH_API_PATH = "/api/health" HEALTH_BROKER_LIVE_COMMAND = "/opt/apache-doris/broker_is_alive.sh" @@ -129,6 +130,11 @@ func NewPodTemplateSpec(dcr *v1.DorisCluster, componentType v1.ComponentType) co volumes = append(volumes, configVolumes...) } + if len(spec.Secrets) != 0 { + secretVolumes, _ := getMultiSecretVolumeAndVolumeMount(spec, componentType) + volumes = append(volumes, secretVolumes...) + } + pts := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: GeneratePodTemplateName(dcr, componentType), @@ -364,6 +370,11 @@ func NewBaseMainContainer(dcr *v1.DorisCluster, config map[string]interface{}, c volumeMounts = append(volumeMounts, configVolumeMounts...) } + if len(spec.Secrets) != 0 { + _, secretVolumeMounts := getMultiSecretVolumeAndVolumeMount(&spec, componentType) + volumeMounts = append(volumeMounts, secretVolumeMounts...) + } + // add basic auth secret volumeMount if dcr.Spec.AuthSecret != "" { volumeMounts = append(volumeMounts, corev1.VolumeMount{ @@ -678,6 +689,46 @@ func getMultiConfigVolumeAndVolumeMount(cmInfo *v1.ConfigMapInfo, componentType return volumes, volumeMounts } +func getMultiSecretVolumeAndVolumeMount(bSpec *v1.BaseSpec, componentType v1.ComponentType) ([]corev1.Volume, []corev1.VolumeMount) { + var volumes []corev1.Volume + var volumeMounts []corev1.VolumeMount + + defaultMountPath := "" + switch componentType { + case v1.Component_FE, v1.Component_BE, v1.Component_CN, v1.Component_Broker: + defaultMountPath = secret_config_path + default: + klog.Infof("getMultiSecretVolumeAndVolumeMount componentType %s not supported.", componentType) + } + + for _, secret := range bSpec.Secrets { + path := secret.MountPath + if secret.MountPath == "" { + path = defaultMountPath + } + volumes = append( + volumes, + corev1.Volume{ + Name: secret.SecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.SecretName, + }, + }, + }, + ) + + volumeMounts = append( + volumeMounts, + corev1.VolumeMount{ + Name: secret.SecretName, + MountPath: path, + }, + ) + } + return volumes, volumeMounts +} + func LivenessProbe(port, timeout int32, path string, commands []string, pt ProbeType) *corev1.Probe { return livenessProbe(port, timeout, path, commands, pt) } diff --git a/pkg/controller/sub_controller/be/controller.go b/pkg/controller/sub_controller/be/controller.go index d6fcfa3..95f8413 100644 --- a/pkg/controller/sub_controller/be/controller.go +++ b/pkg/controller/sub_controller/be/controller.go @@ -73,6 +73,8 @@ func (be *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { } be.CheckConfigMountPath(dcr, v1.Component_BE) + be.CheckSecretMountPath(dcr, v1.Component_BE) + be.CheckSecretExist(ctx, dcr, v1.Component_BE) //generate new be service. svc := resource.BuildExternalService(dcr, v1.Component_BE, config) //create or update be external and domain search service, update the status of fe on src. diff --git a/pkg/controller/sub_controller/broker/controller.go b/pkg/controller/sub_controller/broker/controller.go index 0e87a25..b2d4f66 100644 --- a/pkg/controller/sub_controller/broker/controller.go +++ b/pkg/controller/sub_controller/broker/controller.go @@ -72,6 +72,8 @@ func (bk *Controller) Sync(ctx context.Context, dcr *v1.DorisCluster) error { return err } bk.CheckConfigMountPath(dcr, v1.Component_Broker) + bk.CheckSecretMountPath(dcr, v1.Component_Broker) + bk.CheckSecretExist(ctx, dcr, v1.Component_Broker) internalService := resource.BuildInternalService(dcr, v1.Component_Broker, config) if err := k8s.ApplyService(ctx, bk.K8sclient, &internalService, resource.ServiceDeepEqual); err != nil { klog.Errorf("broker controller sync apply internalService name=%s, namespace=%s, clusterName=%s failed.message=%s.", diff --git a/pkg/controller/sub_controller/cn/controller.go b/pkg/controller/sub_controller/cn/controller.go index 27cfd93..0521d16 100644 --- a/pkg/controller/sub_controller/cn/controller.go +++ b/pkg/controller/sub_controller/cn/controller.go @@ -75,6 +75,8 @@ func (cn *Controller) Sync(ctx context.Context, dcr *dorisv1.DorisCluster) error return err } cn.CheckConfigMountPath(dcr, dorisv1.Component_CN) + cn.CheckSecretMountPath(dcr, dorisv1.Component_CN) + cn.CheckSecretExist(ctx, dcr, dorisv1.Component_CN) svc := resource.BuildExternalService(dcr, dorisv1.Component_CN, config) internalSVC := resource.BuildInternalService(dcr, dorisv1.Component_CN, config) diff --git a/pkg/controller/sub_controller/events.go b/pkg/controller/sub_controller/events.go index 3e3c599..62af1e6 100644 --- a/pkg/controller/sub_controller/events.go +++ b/pkg/controller/sub_controller/events.go @@ -64,6 +64,8 @@ var ( CGStatefulsetDeleteFailed EventReason = "CGStatefulsetDeleteFailed" CGServiceDeleteFailed EventReason = "CGServiceDeleteFailed" ConfigMapPathRepeated EventReason = "ConfigMapPathRepeated" + SecretPathRepeated EventReason = "SecretPathRepeated" + SecretNotExist EventReason = "SecretNotExist" WaitMetaServiceAvailable EventReason = "WaitMetaServiceAvailable" WaitFEAvailable EventReason = "WaitFEAvailable" ServiceApplyedFailed EventReason = "ServiceApplyedFailed" diff --git a/pkg/controller/sub_controller/fe/controller.go b/pkg/controller/sub_controller/fe/controller.go index 1f9f586..726aebe 100644 --- a/pkg/controller/sub_controller/fe/controller.go +++ b/pkg/controller/sub_controller/fe/controller.go @@ -94,6 +94,8 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error return err } fc.CheckConfigMountPath(cluster, v1.Component_FE) + fc.CheckSecretMountPath(cluster, v1.Component_FE) + fc.CheckSecretExist(ctx, cluster, v1.Component_FE) //generate new fe service. svc := resource.BuildExternalService(cluster, v1.Component_FE, config) diff --git a/pkg/controller/sub_controller/sub_controller.go b/pkg/controller/sub_controller/sub_controller.go index 4c93716..aebba2f 100644 --- a/pkg/controller/sub_controller/sub_controller.go +++ b/pkg/controller/sub_controller/sub_controller.go @@ -198,6 +198,60 @@ func (d *SubDefaultController) CheckConfigMountPath(dcr *dorisv1.DorisCluster, c } } +// generate map for mountpath:secret +func (d *SubDefaultController) CheckSecretMountPath(dcr *dorisv1.DorisCluster, componentType dorisv1.ComponentType) { + var secrets []dorisv1.Secret + switch componentType { + case dorisv1.Component_FE: + secrets = dcr.Spec.FeSpec.Secrets + case dorisv1.Component_BE: + secrets = dcr.Spec.BeSpec.Secrets + case dorisv1.Component_CN: + secrets = dcr.Spec.CnSpec.Secrets + case dorisv1.Component_Broker: + secrets = dcr.Spec.BrokerSpec.Secrets + default: + klog.Infof("the componentType %s is not supported.", componentType) + } + var mountsMap = make(map[string]dorisv1.Secret) + for _, secret := range secrets { + path := secret.MountPath + if s, exist := mountsMap[path]; exist { + klog.Errorf("CheckSecretMountPath error: the mountPath %s is repeated between secret: %s and secret: %s.", path, secret.SecretName, s.SecretName) + d.K8srecorder.Event(dcr, string(EventWarning), string(SecretPathRepeated), fmt.Sprintf("the mountPath %s is repeated between secret: %s and secret: %s.", path, secret.SecretName, s.SecretName)) + } + mountsMap[path] = secret + } +} + +// CheckSecretExist, check the secret exist or not in specify namespace. +func (d *SubDefaultController) CheckSecretExist(ctx context.Context, dcr *dorisv1.DorisCluster, componentType dorisv1.ComponentType) { + var secrets []dorisv1.Secret + switch componentType { + case dorisv1.Component_FE: + secrets = dcr.Spec.FeSpec.Secrets + case dorisv1.Component_BE: + secrets = dcr.Spec.BeSpec.Secrets + case dorisv1.Component_CN: + secrets = dcr.Spec.CnSpec.Secrets + case dorisv1.Component_Broker: + secrets = dcr.Spec.BrokerSpec.Secrets + default: + klog.Infof("the componentType %s is not supported.", componentType) + } + errMessage := "" + for _, secret := range secrets { + var s corev1.Secret + if getErr := d.K8sclient.Get(ctx, types.NamespacedName{Namespace: dcr.Namespace, Name: secret.SecretName}, &s); getErr != nil { + errMessage = errMessage + fmt.Sprintf("(name: %s, namespace: %s, err: %s), ", secret.SecretName, dcr.Namespace, getErr.Error()) + } + } + if errMessage != "" { + klog.Errorf("CheckSecretExist error: %s.", errMessage) + d.K8srecorder.Event(dcr, string(EventWarning), string(SecretNotExist), fmt.Sprintf("CheckSecretExist error: %s.", errMessage)) + } +} + // ClearCommonResources clear common resources all component have, as statefulset, service. // response `bool` represents all resource have deleted, if not and delete resource failed return false for next reconcile retry. func (d *SubDefaultController) ClearCommonResources(ctx context.Context, dcr *dorisv1.DorisCluster, componentType dorisv1.ComponentType) (bool, error) {