From db23d42a9b3a58447d6557967bcea938608fca38 Mon Sep 17 00:00:00 2001 From: catpineapple Date: Mon, 11 Nov 2024 18:13:51 +0800 Subject: [PATCH 1/5] add workload group --- api/doris/v1/types.go | 7 +++++++ .../resource/ms_disaggregated_entrypoint.sh | 2 +- pkg/common/utils/resource/constants.go | 2 ++ pkg/controller/sub_controller/be/pod.go | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/api/doris/v1/types.go b/api/doris/v1/types.go index c1b9512..7dc18ea 100644 --- a/api/doris/v1/types.go +++ b/api/doris/v1/types.go @@ -88,6 +88,13 @@ type FeSpec struct { type BeSpec struct { //the foundation spec for creating be software services. BaseSpec `json:",inline"` + + // EnableWorkloadGroup is a switch that determines whether the doris cluster enables the workload group. + // Default value is 'false'. + // Enabling it means that the container must be started in privileged mode. + // Please confirm whether the host machine and k8s cluster allow it. + // Doris workloadgroup reference document: https://doris.apache.org/docs/admin-manual/resource-admin/workload-group + EnableWorkloadGroup bool `json:"enableWorkloadGroup,omitempty"` } // FeAddress specify the fe address, please set it when you deploy fe outside k8s or deploy components use crd except fe, if not set . diff --git a/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh b/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh index a0bd8f6..6fdb0b1 100755 --- a/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh +++ b/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh @@ -41,4 +41,4 @@ if [[ -d $CONFIGMAP_PATH ]]; then done fi -$DORIS_HOME/ms/bin/start.sh +$DORIS_HOME/ms/bin/start.sh --console diff --git a/pkg/common/utils/resource/constants.go b/pkg/common/utils/resource/constants.go index 7db3b03..72f88c8 100644 --- a/pkg/common/utils/resource/constants.go +++ b/pkg/common/utils/resource/constants.go @@ -23,6 +23,8 @@ const ( ENV_FE_PORT = "FE_QUERY_PORT" + ENABLE_WORKLOAD_GROUP = "ENABLE_WORKLOAD_GROUP" + ENV_FE_ELECT_NUMBER = "ELECT_NUMBER" COMPONENT_TYPE = "COMPONENT_TYPE" diff --git a/pkg/controller/sub_controller/be/pod.go b/pkg/controller/sub_controller/be/pod.go index a360247..f67c697 100644 --- a/pkg/controller/sub_controller/be/pod.go +++ b/pkg/controller/sub_controller/be/pod.go @@ -19,6 +19,7 @@ package be import ( "context" + "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "strconv" @@ -36,7 +37,16 @@ func (be *Controller) buildBEPodTemplateSpec(dcr *v1.DorisCluster) corev1.PodTem containers = append(containers, podTemplateSpec.Spec.Containers...) beContainer := be.beContainer(dcr) containers = append(containers, beContainer) + + if dcr.Spec.BeSpec.EnableWorkloadGroup { + privileged := true + if dcr.Spec.BeSpec.ContainerSecurityContext == nil { + dcr.Spec.BeSpec.ContainerSecurityContext = &corev1.SecurityContext{} + } + dcr.Spec.BeSpec.ContainerSecurityContext.Privileged = &privileged + } containers = resource.ApplySecurityContext(containers, dcr.Spec.BeSpec.ContainerSecurityContext) + podTemplateSpec.Spec.Containers = containers return podTemplateSpec } @@ -96,6 +106,13 @@ func (be *Controller) beContainer(dcr *v1.DorisCluster) corev1.Container { Value: feQueryPort, }) + if dcr.Spec.BeSpec.EnableWorkloadGroup { + c.Env = append(c.Env, corev1.EnvVar{ + Name: resource.ENABLE_WORKLOAD_GROUP, + Value: fmt.Sprintf("%t", dcr.Spec.BeSpec.EnableWorkloadGroup), + }) + } + return c } From 5a0594ccca2f7458b7d693c6e845af640fd7dacf Mon Sep 17 00:00:00 2001 From: catpineapple Date: Mon, 11 Nov 2024 20:10:25 +0800 Subject: [PATCH 2/5] modify entrypoint --- docker/disaggregate/resource/be_entrypoint.sh | 285 ++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100755 docker/disaggregate/resource/be_entrypoint.sh diff --git a/docker/disaggregate/resource/be_entrypoint.sh b/docker/disaggregate/resource/be_entrypoint.sh new file mode 100755 index 0000000..db519d3 --- /dev/null +++ b/docker/disaggregate/resource/be_entrypoint.sh @@ -0,0 +1,285 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# the fe query port for mysql. +FE_QUERY_PORT=${FE_QUERY_PORT:-9030} +# timeout for probe fe master. +PROBE_TIMEOUT=60 +# interval time to probe fe. +PROBE_INTERVAL=2 +# rpc port for fe communicate with be. +HEARTBEAT_PORT=9050 +# fqdn or ip +MY_SELF= +MY_IP=`hostname -i` +MY_HOSTNAME=`hostname -f` +DORIS_ROOT=${DORIS_ROOT:-"/opt/apache-doris"} +# if config secret for basic auth about operate node of doris, the path must be `/etc/basic_auth`. This is set by operator and the key of password must be `password`. +AUTH_PATH="/etc/basic_auth" +DORIS_HOME=${DORIS_ROOT}/be +BE_CONFIG=$DORIS_HOME/conf/be.conf +# represents self in fe meta or not. +REGISTERED=false + +DB_ADMIN_USER=${USER:-"root"} + +DB_ADMIN_PASSWD=$PASSWD + +ENABLE_WORKLOAD_GROUP=${ENABLE_WORKLOAD_GROUP:-false} +WORKLOAD_GROUP_PATH="/sys/fs/cgroup/cpu/doris" + + +log_stderr() +{ + echo "[`date`] $@" >&2 +} + +function add_cluster_info_to_conf() +{ + if [[ "x$ENABLE_WORKLOAD_GROUP" == "xtrue" ]]; then + echo "doris_cgroup_cpu_path=$WORKLOAD_GROUP_PATH" >> ${DORIS_HOME}/conf/be.conf + fi +} + +update_conf_from_configmap() +{ + add_cluster_info_to_conf + if [[ "x$CONFIGMAP_MOUNT_PATH" == "x" ]] ; then + log_stderr '[info] Empty $CONFIGMAP_MOUNT_PATH env var, skip it!' + return 0 + fi + if ! test -d $CONFIGMAP_MOUNT_PATH ; then + log_stderr "[info] $CONFIGMAP_MOUNT_PATH not exist or not a directory, ignore ..." + return 0 + fi + local tgtconfdir=$DORIS_HOME/conf + for conffile in `ls $CONFIGMAP_MOUNT_PATH` + do + log_stderr "[info] Process conf file $conffile ..." + local tgt=$tgtconfdir/$conffile + if test -e $tgt ; then + # make a backup + mv -f $tgt ${tgt}.bak + fi + if [[ "$conffile" == "be.conf" ]]; then + cp $CONFIGMAP_MOUNT_PATH/$conffile $DORIS_HOME/conf/$file + add_cluster_info_to_conf + continue + fi + ln -sfT $CONFIGMAP_MOUNT_PATH/$conffile $tgt + done +} + +# resolve password for root +resolve_password_from_secret() +{ + if [[ -f "$AUTH_PATH/password" ]]; then + DB_ADMIN_PASSWD=`cat $AUTH_PATH/password` + fi + if [[ -f "$AUTH_PATH/username" ]]; then + DB_ADMIN_USER=`cat $AUTH_PATH/username` + fi +} + +# get all backends info to check self exist or not. +show_backends(){ + local svc=$1 + backends=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e 'SHOW BACKENDS;' 2>&1` + log_stderr "[info] use root no password show backends result $backends ." + if echo $backends | grep -w "1045" | grep -q -w "28000" &>/dev/null; then + log_stderr "[info] use username and password that configured to show backends." + backends=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e 'SHOW BACKENDS;'` + fi + + echo "$backends" +} + +# get all registered fe in cluster, for check the fe have `MASTER`. +function show_frontends() +{ + local addr=$1 + frontends=`timeout 15 mysql --connect-timeout 2 -h $addr -P $FE_QUERY_PORT -uroot --batch -e 'show frontends;' 2>&1` + log_stderr "[info] use root no password show frontends result $frontends ." + if echo $frontends | grep -w "1045" | grep -q -w "28000" &>/dev/null; then + log_stderr "[info] use username and passwore that configured to show frontends." + frontends=`timeout 15 mysql --connect-timeout 2 -h $addr -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --batch -e 'show frontends;'` + fi + + echo "$frontends" +} + +#parse the `$BE_CONFIG` file, passing the key need resolve as parameter. +parse_confval_from_conf() +{ + # a naive script to grep given confkey from fe conf file + # assume conf format: ^\s*\s*=\s*\s*$ + local confkey=$1 + local confvalue=`grep "\<$confkey\>" $BE_CONFIG | grep -v '^\s*#' | sed 's|^\s*'$confkey'\s*=\s*\(.*\)\s*$|\1|g'` + echo "$confvalue" +} + +collect_env_info() +{ + # heartbeat_port from conf file + local heartbeat_port=`parse_confval_from_conf "heartbeat_service_port"` + if [[ "x$heartbeat_port" != "x" ]] ; then + HEARTBEAT_PORT=$heartbeat_port + fi + + if [[ "x$HOST_TYPE" == "xIP" ]] ; then + MY_SELF=$MY_IP + else + MY_SELF=$MY_HOSTNAME + fi +} + +add_self() +{ + local svc=$1 + start=`date +%s` + local timeout=$PROBE_TIMEOUT + + while true + do + memlist=`show_backends $svc` + if echo "$memlist" | grep -q -w "$MY_SELF" &>/dev/null ; then + log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) exist in FE, start be directly ..." + break; + fi + + # check fe cluster have master, if fe have not master wait. + fe_memlist=`show_frontends $svc` + local pos=`echo "$fe_memlist" | grep '\' | awk -F '\t' '{for(i=1;i' | awk -v p="$pos" -F '\t' '{if ($p=="true") print $2}'` + log_stderr "'IsMaster' sequence in columns is $pos master=$leader ." + + if [[ "x$leader" == "x" ]]; then + log_stderr "[info] resolve the eighth column for finding master !" + leader=`echo "$fe_memlist" | grep '\' | awk -F '\t' '{if ($8=="true") print $2}'` + fi + if [[ "x$leader" == "x" ]]; then + # compatible 2.1.0 + log_stderr "[info] resoluve the ninth column for finding master!" + leader=`echo "$fe_memlist" | grep '\' | awk -F '\t' '{if ($9=="true") print $2}'` + fi + + if [[ "x$leader" != "x" ]]; then + create_account $leader + log_stderr "[info] myself ($MY_SELF:$HEARTBEAT_PORT) not exist in FE and fe have leader register myself into fe." + add_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM ADD BACKEND \"$MY_SELF:$HEARTBEAT_PORT\";" 2>&1` + if echo $add_result | grep -w "1045" | grep -q -w "28000" &>/dev/null ; then + timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM ADD BACKEND \"$MY_SELF:$HEARTBEAT_PORT\";" + fi + + let "expire=start+timeout" + now=`date +%s` + if [[ $expire -le $now ]] ; then + log_stderr "[error] exit probe master for probing timeout." + return 0 + fi + else + log_stderr "[info] not have leader wait fe cluster elect a master, sleep 2s..." + sleep $PROBE_INTERVAL + fi + done +} + +function create_account() +{ + master=$1 + users=`mysql --connect-timeout 2 -h $master -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e 'SHOW ALL GRANTS;' 2>&1` + if echo $users | grep -w "1045" | grep -q -w "28000" &>/dev/null; then + log_stderr "the 'root' account have set password! not need auto create management account." + return 0 + fi + if echo $users | grep -q -w "$DB_ADMIN_USER" &>/dev/null; then + log_stderr "the $DB_ADMIN_USER have exist in doris." + return 0 + fi + mysql --connect-timeout 2 -h $master -P$FE_QUERY_PORT -uroot --skip-column-names --batch -e "CREATE USER '$DB_ADMIN_USER' IDENTIFIED BY '$DB_ADMIN_PASSWD';GRANT NODE_PRIV ON *.*.* TO $DB_ADMIN_USER;" 2>&1 + log_stderr "created new account and grant NODE_PRIV!" + +} + +# check be exist or not, if exist return 0, or register self in fe cluster. when all fe address failed exit script. +# `xxx1:port,xxx2:port` as parameter to function. +function check_and_register() +{ + addrs=$1 + local addrArr=(${addrs//,/ }) + for addr in ${addrArr[@]} + do + add_self $addr + + if [[ $REGISTERED ]]; then + break; + fi + done + + if [[ $REGISTERED ]]; then + return 0 + else + log_stderr "not find master in fe cluster, please use mysql connect to fe for verfing the master exist and verify domain connectivity with two pods in different node. " + exit 1 + fi +} + +function work_load_group_for_cgroup_path() { + output=$(cat /proc/filesystems | grep cgroup) + if [ -z "$output" ]; then + log_stderr "[error] The host machine does not have cgroup installed, so the workload group function will be limited." + exit 1 + fi + + mkdir -p /sys/fs/cgroup/cpu/doris + chmod 770 /sys/fs/cgroup/cpu/doris + chown -R root:root /sys/fs/cgroup/cpu/doris + + if [[ -f "/sys/fs/cgroup/cgroup.controllers" ]]; then + log_stderr "[info] The host machine cgroup version: v2." + chmod a+w /sys/fs/cgroup/cgroup.procs + else + log_stderr "[info] The host machine cgroup version: v1." + fi +} + +fe_addrs=$1 +if [[ "x$fe_addrs" == "x" ]]; then + echo "need fe address as paramter!" + echo " Example $0 " + exit 1 +fi + +if [[ "x$ENABLE_WORKLOAD_GROUP" == "xtrue" ]]; then + log_stderr '[info] Enable workload group !' + work_load_group_for_cgroup_path +fi + +update_conf_from_configmap +# resolve password for root to manage nodes in doris. +resolve_password_from_secret +collect_env_info +#add_self $fe_addr || exit $? +check_and_register $fe_addrs +./doris-debug --component be +log_stderr "run start_be.sh" +# the server will start in the current terminal session, and the log output and console interaction will be printed to that terminal +# befor doris 2.0.2 ,doris start with : start_xx.sh +# sine doris 2.0.2 ,doris start with : start_xx.sh --console doc: https://doris.apache.org/docs/dev/install/standard-deployment/#version--202 +$DORIS_HOME/bin/start_be.sh --console \ No newline at end of file From be9872156b034bb8839fdbb775ddcdd4d32616e8 Mon Sep 17 00:00:00 2001 From: catpineapple Date: Tue, 12 Nov 2024 14:19:03 +0800 Subject: [PATCH 3/5] use point util --- pkg/controller/sub_controller/be/pod.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/sub_controller/be/pod.go b/pkg/controller/sub_controller/be/pod.go index f67c697..a0b6b8b 100644 --- a/pkg/controller/sub_controller/be/pod.go +++ b/pkg/controller/sub_controller/be/pod.go @@ -21,6 +21,7 @@ import ( "context" "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "strconv" v1 "github.com/apache/doris-operator/api/doris/v1" @@ -39,11 +40,10 @@ func (be *Controller) buildBEPodTemplateSpec(dcr *v1.DorisCluster) corev1.PodTem containers = append(containers, beContainer) if dcr.Spec.BeSpec.EnableWorkloadGroup { - privileged := true if dcr.Spec.BeSpec.ContainerSecurityContext == nil { dcr.Spec.BeSpec.ContainerSecurityContext = &corev1.SecurityContext{} } - dcr.Spec.BeSpec.ContainerSecurityContext.Privileged = &privileged + dcr.Spec.BeSpec.ContainerSecurityContext.Privileged = pointer.Bool(true) } containers = resource.ApplySecurityContext(containers, dcr.Spec.BeSpec.ContainerSecurityContext) From d33b95665f42691fcb4b996b38e332e24bfd8893 Mon Sep 17 00:00:00 2001 From: catpineapple Date: Tue, 12 Nov 2024 14:20:44 +0800 Subject: [PATCH 4/5] shell modify --- docker/disaggregate/resource/be_entrypoint.sh | 285 ------------------ .../resource/ms_disaggregated_entrypoint.sh | 2 +- 2 files changed, 1 insertion(+), 286 deletions(-) delete mode 100755 docker/disaggregate/resource/be_entrypoint.sh diff --git a/docker/disaggregate/resource/be_entrypoint.sh b/docker/disaggregate/resource/be_entrypoint.sh deleted file mode 100755 index db519d3..0000000 --- a/docker/disaggregate/resource/be_entrypoint.sh +++ /dev/null @@ -1,285 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# the fe query port for mysql. -FE_QUERY_PORT=${FE_QUERY_PORT:-9030} -# timeout for probe fe master. -PROBE_TIMEOUT=60 -# interval time to probe fe. -PROBE_INTERVAL=2 -# rpc port for fe communicate with be. -HEARTBEAT_PORT=9050 -# fqdn or ip -MY_SELF= -MY_IP=`hostname -i` -MY_HOSTNAME=`hostname -f` -DORIS_ROOT=${DORIS_ROOT:-"/opt/apache-doris"} -# if config secret for basic auth about operate node of doris, the path must be `/etc/basic_auth`. This is set by operator and the key of password must be `password`. -AUTH_PATH="/etc/basic_auth" -DORIS_HOME=${DORIS_ROOT}/be -BE_CONFIG=$DORIS_HOME/conf/be.conf -# represents self in fe meta or not. -REGISTERED=false - -DB_ADMIN_USER=${USER:-"root"} - -DB_ADMIN_PASSWD=$PASSWD - -ENABLE_WORKLOAD_GROUP=${ENABLE_WORKLOAD_GROUP:-false} -WORKLOAD_GROUP_PATH="/sys/fs/cgroup/cpu/doris" - - -log_stderr() -{ - echo "[`date`] $@" >&2 -} - -function add_cluster_info_to_conf() -{ - if [[ "x$ENABLE_WORKLOAD_GROUP" == "xtrue" ]]; then - echo "doris_cgroup_cpu_path=$WORKLOAD_GROUP_PATH" >> ${DORIS_HOME}/conf/be.conf - fi -} - -update_conf_from_configmap() -{ - add_cluster_info_to_conf - if [[ "x$CONFIGMAP_MOUNT_PATH" == "x" ]] ; then - log_stderr '[info] Empty $CONFIGMAP_MOUNT_PATH env var, skip it!' - return 0 - fi - if ! test -d $CONFIGMAP_MOUNT_PATH ; then - log_stderr "[info] $CONFIGMAP_MOUNT_PATH not exist or not a directory, ignore ..." - return 0 - fi - local tgtconfdir=$DORIS_HOME/conf - for conffile in `ls $CONFIGMAP_MOUNT_PATH` - do - log_stderr "[info] Process conf file $conffile ..." - local tgt=$tgtconfdir/$conffile - if test -e $tgt ; then - # make a backup - mv -f $tgt ${tgt}.bak - fi - if [[ "$conffile" == "be.conf" ]]; then - cp $CONFIGMAP_MOUNT_PATH/$conffile $DORIS_HOME/conf/$file - add_cluster_info_to_conf - continue - fi - ln -sfT $CONFIGMAP_MOUNT_PATH/$conffile $tgt - done -} - -# resolve password for root -resolve_password_from_secret() -{ - if [[ -f "$AUTH_PATH/password" ]]; then - DB_ADMIN_PASSWD=`cat $AUTH_PATH/password` - fi - if [[ -f "$AUTH_PATH/username" ]]; then - DB_ADMIN_USER=`cat $AUTH_PATH/username` - fi -} - -# get all backends info to check self exist or not. -show_backends(){ - local svc=$1 - backends=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e 'SHOW BACKENDS;' 2>&1` - log_stderr "[info] use root no password show backends result $backends ." - if echo $backends | grep -w "1045" | grep -q -w "28000" &>/dev/null; then - log_stderr "[info] use username and password that configured to show backends." - backends=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e 'SHOW BACKENDS;'` - fi - - echo "$backends" -} - -# get all registered fe in cluster, for check the fe have `MASTER`. -function show_frontends() -{ - local addr=$1 - frontends=`timeout 15 mysql --connect-timeout 2 -h $addr -P $FE_QUERY_PORT -uroot --batch -e 'show frontends;' 2>&1` - log_stderr "[info] use root no password show frontends result $frontends ." - if echo $frontends | grep -w "1045" | grep -q -w "28000" &>/dev/null; then - log_stderr "[info] use username and passwore that configured to show frontends." - frontends=`timeout 15 mysql --connect-timeout 2 -h $addr -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --batch -e 'show frontends;'` - fi - - echo "$frontends" -} - -#parse the `$BE_CONFIG` file, passing the key need resolve as parameter. -parse_confval_from_conf() -{ - # a naive script to grep given confkey from fe conf file - # assume conf format: ^\s*\s*=\s*\s*$ - local confkey=$1 - local confvalue=`grep "\<$confkey\>" $BE_CONFIG | grep -v '^\s*#' | sed 's|^\s*'$confkey'\s*=\s*\(.*\)\s*$|\1|g'` - echo "$confvalue" -} - -collect_env_info() -{ - # heartbeat_port from conf file - local heartbeat_port=`parse_confval_from_conf "heartbeat_service_port"` - if [[ "x$heartbeat_port" != "x" ]] ; then - HEARTBEAT_PORT=$heartbeat_port - fi - - if [[ "x$HOST_TYPE" == "xIP" ]] ; then - MY_SELF=$MY_IP - else - MY_SELF=$MY_HOSTNAME - fi -} - -add_self() -{ - local svc=$1 - start=`date +%s` - local timeout=$PROBE_TIMEOUT - - while true - do - memlist=`show_backends $svc` - if echo "$memlist" | grep -q -w "$MY_SELF" &>/dev/null ; then - log_stderr "[info] Check myself ($MY_SELF:$HEARTBEAT_PORT) exist in FE, start be directly ..." - break; - fi - - # check fe cluster have master, if fe have not master wait. - fe_memlist=`show_frontends $svc` - local pos=`echo "$fe_memlist" | grep '\' | awk -F '\t' '{for(i=1;i' | awk -v p="$pos" -F '\t' '{if ($p=="true") print $2}'` - log_stderr "'IsMaster' sequence in columns is $pos master=$leader ." - - if [[ "x$leader" == "x" ]]; then - log_stderr "[info] resolve the eighth column for finding master !" - leader=`echo "$fe_memlist" | grep '\' | awk -F '\t' '{if ($8=="true") print $2}'` - fi - if [[ "x$leader" == "x" ]]; then - # compatible 2.1.0 - log_stderr "[info] resoluve the ninth column for finding master!" - leader=`echo "$fe_memlist" | grep '\' | awk -F '\t' '{if ($9=="true") print $2}'` - fi - - if [[ "x$leader" != "x" ]]; then - create_account $leader - log_stderr "[info] myself ($MY_SELF:$HEARTBEAT_PORT) not exist in FE and fe have leader register myself into fe." - add_result=`timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e "ALTER SYSTEM ADD BACKEND \"$MY_SELF:$HEARTBEAT_PORT\";" 2>&1` - if echo $add_result | grep -w "1045" | grep -q -w "28000" &>/dev/null ; then - timeout 15 mysql --connect-timeout 2 -h $svc -P $FE_QUERY_PORT -u$DB_ADMIN_USER -p$DB_ADMIN_PASSWD --skip-column-names --batch -e "ALTER SYSTEM ADD BACKEND \"$MY_SELF:$HEARTBEAT_PORT\";" - fi - - let "expire=start+timeout" - now=`date +%s` - if [[ $expire -le $now ]] ; then - log_stderr "[error] exit probe master for probing timeout." - return 0 - fi - else - log_stderr "[info] not have leader wait fe cluster elect a master, sleep 2s..." - sleep $PROBE_INTERVAL - fi - done -} - -function create_account() -{ - master=$1 - users=`mysql --connect-timeout 2 -h $master -P $FE_QUERY_PORT -uroot --skip-column-names --batch -e 'SHOW ALL GRANTS;' 2>&1` - if echo $users | grep -w "1045" | grep -q -w "28000" &>/dev/null; then - log_stderr "the 'root' account have set password! not need auto create management account." - return 0 - fi - if echo $users | grep -q -w "$DB_ADMIN_USER" &>/dev/null; then - log_stderr "the $DB_ADMIN_USER have exist in doris." - return 0 - fi - mysql --connect-timeout 2 -h $master -P$FE_QUERY_PORT -uroot --skip-column-names --batch -e "CREATE USER '$DB_ADMIN_USER' IDENTIFIED BY '$DB_ADMIN_PASSWD';GRANT NODE_PRIV ON *.*.* TO $DB_ADMIN_USER;" 2>&1 - log_stderr "created new account and grant NODE_PRIV!" - -} - -# check be exist or not, if exist return 0, or register self in fe cluster. when all fe address failed exit script. -# `xxx1:port,xxx2:port` as parameter to function. -function check_and_register() -{ - addrs=$1 - local addrArr=(${addrs//,/ }) - for addr in ${addrArr[@]} - do - add_self $addr - - if [[ $REGISTERED ]]; then - break; - fi - done - - if [[ $REGISTERED ]]; then - return 0 - else - log_stderr "not find master in fe cluster, please use mysql connect to fe for verfing the master exist and verify domain connectivity with two pods in different node. " - exit 1 - fi -} - -function work_load_group_for_cgroup_path() { - output=$(cat /proc/filesystems | grep cgroup) - if [ -z "$output" ]; then - log_stderr "[error] The host machine does not have cgroup installed, so the workload group function will be limited." - exit 1 - fi - - mkdir -p /sys/fs/cgroup/cpu/doris - chmod 770 /sys/fs/cgroup/cpu/doris - chown -R root:root /sys/fs/cgroup/cpu/doris - - if [[ -f "/sys/fs/cgroup/cgroup.controllers" ]]; then - log_stderr "[info] The host machine cgroup version: v2." - chmod a+w /sys/fs/cgroup/cgroup.procs - else - log_stderr "[info] The host machine cgroup version: v1." - fi -} - -fe_addrs=$1 -if [[ "x$fe_addrs" == "x" ]]; then - echo "need fe address as paramter!" - echo " Example $0 " - exit 1 -fi - -if [[ "x$ENABLE_WORKLOAD_GROUP" == "xtrue" ]]; then - log_stderr '[info] Enable workload group !' - work_load_group_for_cgroup_path -fi - -update_conf_from_configmap -# resolve password for root to manage nodes in doris. -resolve_password_from_secret -collect_env_info -#add_self $fe_addr || exit $? -check_and_register $fe_addrs -./doris-debug --component be -log_stderr "run start_be.sh" -# the server will start in the current terminal session, and the log output and console interaction will be printed to that terminal -# befor doris 2.0.2 ,doris start with : start_xx.sh -# sine doris 2.0.2 ,doris start with : start_xx.sh --console doc: https://doris.apache.org/docs/dev/install/standard-deployment/#version--202 -$DORIS_HOME/bin/start_be.sh --console \ No newline at end of file diff --git a/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh b/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh index 6fdb0b1..8fd691f 100755 --- a/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh +++ b/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh @@ -41,4 +41,4 @@ if [[ -d $CONFIGMAP_PATH ]]; then done fi -$DORIS_HOME/ms/bin/start.sh --console +$DORIS_HOME/ms/bin/start.sh \ No newline at end of file From 34930e54d53b7e7b8d3868ac0c06e5bc2f4eb5b2 Mon Sep 17 00:00:00 2001 From: catpineapple Date: Tue, 12 Nov 2024 14:29:38 +0800 Subject: [PATCH 5/5] shell modify --- docker/disaggregate/resource/ms_disaggregated_entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh b/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh index 8fd691f..a0bd8f6 100755 --- a/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh +++ b/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh @@ -41,4 +41,4 @@ if [[ -d $CONFIGMAP_PATH ]]; then done fi -$DORIS_HOME/ms/bin/start.sh \ No newline at end of file +$DORIS_HOME/ms/bin/start.sh