diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 526aab2a..23fe1452 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -1,25 +1,24 @@ -app_version: 3.0.1 +app_version: 3.1.0 gcr_mirror: kubesphere -etcd_version: 3.3.12 +etcd_version: 3.4.13 cni_version: 0.8.6 -k8s_version: 1.18.12 +k8s_version: 1.19.8 flannel_version: 0.12.0 coredns_version: 1.6.9 -calico_version: 3.15.1 -hostnic_version: 1.0.0 -qingcloud_csi_version: 1.2.6 -qingcloud_ccm_version: 1.4.5 -ks_version: 3.0.0 -ks_installer_image_tag: qke-3.0.1 +calico_version: 3.16.3 +hostnic_version: 1.0.0-alpha.5 +qingcloud_csi_version: 1.2.7 +qingcloud_ccm_version: 1.4.7 +ks_version: 3.1.0 +ks_installer_image_tag: v3.1.0 helm_version: 3.2.1 helm_2to3_version: 0.8.0 helm_stable_repo: https://charts.kubesphere.io/mirror helm_main_repo: https://charts.kubesphere.io/main helm_test_repo: https://charts.kubesphere.io/test -nodelocaldns_version: 1.15.5 -nvidia_tesla_version: 418.116.00 +nodelocaldns_version: 1.15.12 +nvidia_tesla_version: 460.73.01 nvidia_plugin_version: 1.0.0-beta4 -# 本地 images package 的位置 local_images_path: /images ks_images_package: allks.tgz k8s_images_package: allk8s.tgz @@ -33,51 +32,6 @@ binaries: - /opt/k8s docker_images_k8s: -- kubesphere/cloud-controller-manager:v1.4.4 - -docker_images_ks: -- alpine:3.10.4 -- docker:19.03 -- haproxy:2.0.4 -- jenkins/jenkins:2.176.2 -- kubesphere/builder-base:v2.1.0 -- kubesphere/builder-go:v2.1.0 -- kubesphere/builder-maven:v2.1.0 -- kubesphere/builder-nodejs:v2.1.0 -- kubesphere/elasticsearch-oss:6.7.0-1 -- kubesphere/etcd:v3.2.18 -- kubesphere/java-11-centos7:v2.1.0 -- kubesphere/java-11-runtime:v2.1.0 -- kubesphere/java-8-centos7:v2.1.0 -- kubesphere/java-8-runtime:v2.1.0 -- kubesphere/kubectl:v1.0.0 -- kubesphere/kube-rbac-proxy:v0.4.1 -- kubesphere/nodejs-4-centos7:v2.1.0 -- kubesphere/nodejs-6-centos7:v2.1.0 -- kubesphere/nodejs-8-centos7:v2.1.0 -- kubesphere/python-27-centos7:v2.1.0 -- kubesphere/python-34-centos7:v2.1.0 -- kubesphere/python-35-centos7:v2.1.0 -- kubesphere/python-36-centos7:v2.1.0 -- kubesphere/s2i-binary:v2.1.0 -- kubesphere/s2ioperator:v2.1.1 -- kubesphere/s2irun:v2.1.1 -- kubesphere/tomcat85-java11-centos7:v2.1.0 -- kubesphere/tomcat85-java11-runtime:v2.1.0 -- kubesphere/tomcat85-java8-centos7:v2.1.0 -- kubesphere/tomcat85-java8-runtime:v2.1.0 -- minio/mc:RELEASE.2019-08-07T23-14-43Z -- minio/minio:RELEASE.2019-08-07T01-59-21Z -- mysql:8.0.11 -- nginx:1.14-alpine -- osixia/openldap:1.3.0 -- redis:5.0.5-alpine - -docker_images_k8s_new: -- calico/cni:v3.15.1 -- calico/kube-controllers:v3.15.1 -- calico/node:v3.15.1 -- calico/pod2daemon-flexvol:v3.15.1 - coredns/coredns:1.6.9 - csiplugin/csi-attacher:v2.1.1 - csiplugin/csi-node-driver-registrar:v1.2.0 @@ -86,23 +40,11 @@ docker_images_k8s_new: - csiplugin/csi-resizer:v0.4.0 - csiplugin/csi-snapshotter:v2.0.1 - csiplugin/snapshot-controller:v2.0.1 -- kubesphere/hostnic:v1.0.0 - nvidia/k8s-device-plugin:1.0.0-beta4 - kubesphere/flannel:v0.12.0 -- kubesphere/kube-apiserver:v1.18.12 -- kubesphere/kube-controller-manager:v1.18.12 -- kubesphere/kube-proxy:v1.18.12 -- kubesphere/kube-scheduler:v1.18.12 -- kubesphere/pause:3.2 +- kubesphere/pause:3.1 -docker_images_ks_new: -- istio/citadel:1.4.8 -- istio/galley:1.4.8 -- istio/kubectl:1.4.8 -- istio/mixer:1.4.8 -- istio/pilot:1.4.8 -- istio/proxyv2:1.4.8 -- istio/sidecar_injector:1.4.8 +docker_images_ks: - jaegertracing/jaeger-agent:1.17 - jaegertracing/jaeger-collector:1.17 - jaegertracing/jaeger-es-index-cleaner:1.17.1 @@ -110,42 +52,110 @@ docker_images_ks_new: - jaegertracing/jaeger-query:1.17 - jenkins/jnlp-slave:3.27-1 - jimmidyson/configmap-reload:v0.3.0 -- kubesphere/alert-adapter:v3.0.0 -- kubesphere/alerting-dbinit:v3.0.0 -- kubesphere/alerting:v2.1.2 - kubesphere/fluentbit-operator:migrator -- kubesphere/fluentbit-operator:v0.2.0 -- kubesphere/fluent-bit:v1.4.6 -- kubesphere/jenkins-uc:v3.0.0 -- kubesphere/ks-apiserver:v3.0.0 -- kubesphere/ks-console:v3.0.0 -- kubesphere/ks-controller-manager:v3.0.0 -- kubesphere/ks-devops:flyway-v3.0.0 -- kubesphere/ks-installer:qke-3.0.0 -- kubesphere/ks-upgrade:v3.0.0 - kubesphere/kube-auditing-operator:v0.1.2 - kubesphere/kube-auditing-webhook:v0.1.2 - kubesphere/kube-events-exporter:v0.1.0 - kubesphere/kube-events-operator:v0.1.0 -- kubesphere/kube-events-ruler:v0.1.0 -- kubesphere/kubefed:v0.3.0 -- kubesphere/kube-state-metrics:v1.9.6 - kubesphere/log-sidecar-injector:1.1 - kubesphere/elasticsearch-curator:v5.7.6 -- kubesphere/metrics-server:v0.3.7 - kubesphere/nginx-ingress-controller:0.24.1 - kubesphere/node-exporter:ks-v0.18.1 -- kubesphere/notification:flyway_v2.1.2 -- kubesphere/notification-manager-operator:v0.1.0 -- kubesphere/notification-manager:v0.1.0 -- kubesphere/notification:v2.1.2 -- kubesphere/prometheus-config-reloader:v0.38.3 -- kubesphere/prometheus-operator:v0.38.3 -- kubesphere/tower:v0.1.0 - mirrorgooglecontainers/defaultbackend-amd64:1.4 -- openpitrix/generate-kubeconfig:v0.5.0 -- openpitrix/openpitrix:flyway-v0.5.0 -- openpitrix/openpitrix:v0.5.0 -- openpitrix/release-app:sha-303629d - prom/alertmanager:v0.21.0 -- prom/prometheus:v2.20.1 +- docker:19.03 + +docker_images_k8s_new: +- kubesphere/kube-apiserver:v1.19.8 +- kubesphere/kube-scheduler:v1.19.8 +- kubesphere/kube-proxy:v1.19.8 +- kubesphere/kube-controller-manager:v1.19.8 +- kubesphere/pause:3.2 +- kubesphere/etcd:v3.4.13 +- calico/cni:v3.16.3 +- calico/kube-controllers:v3.16.3 +- calico/node:v3.16.3 +- calico/pod2daemon-flexvol:v3.16.3 +- kubesphere/k8s-dns-node-cache:1.15.12 +- kubesphere/nfs-client-provisioner:v3.1.0-k8s1.11 +- csiplugin/csi-qingcloud:v1.2.1 +- csiplugin/csi-neonsan:v1.2.1 +- csiplugin/csi-neonsan-ubuntu:v1.2.1 +- csiplugin/csi-neonsan-centos:v1.2.1 +- kubesphere/cloud-controller-manager:v1.4.7 +- kubesphere/hostnic:v1.0.0-alpha.5 +- kubesphere/metrics-server:v0.4.2 + +docker_images_ks_new: +- kubesphere/ks-apiserver:v3.1.0 +- kubesphere/ks-console:v3.1.0 +- kubesphere/ks-controller-manager:v3.1.0 +- kubesphere/ks-installer:v3.1.0 +- kubesphere/kubectl:v1.19.0 +- kubesphere/nginx-ingress-controller:v0.35.0 +- kubesphere/kubefed:v0.7.0 +- kubesphere/tower:v0.2.0 +- kubesphere/prometheus-config-reloader:v0.42.1 +- kubesphere/prometheus-operator:v0.42.1 +- kubesphere/kube-state-metrics:v1.9.7 +- prom/prometheus:v2.26.0 +- prom/node-exporter:v0.18.1 +- kubesphere/ks-alerting-migration:v3.1.0 +- kubesphere/notification-manager-operator:v1.0.0 +- kubesphere/notification-manager:v1.0.0 +- kubesphere/kube-rbac-proxy:v0.8.0 +- openebs/provisioner-localpv:2.3.0 +- thanosio/thanos:v0.18.0 +- grafana/grafana:7.4.3 +- kubesphere/fluentbit-operator:v0.5.0 +- kubesphere/fluent-bit:v1.6.9 +- elastic/filebeat:6.7.0 +- kubesphere/kube-events-ruler:v0.2.0 +- istio/pilot:1.6.10 +- istio/proxyv2:1.6.10 +- kubesphere/kiali:v1.26.1 +- kubesphere/kiali-operator:v1.26.1 +- kubesphere/ks-jenkins:2.249.1 +- kubesphere/s2ioperator:v3.1.0 +- kubesphere/openpitrix-jobs:v3.1.0 +- weaveworks/scope:1.13.0 +- kubeedge/cloudcore:v1.6.1 +- kubesphere/edge-watcher:v0.1.0 +- kubesphere/kube-rbac-proxy:v0.5.0 +- kubesphere/edge-watcher-agent:v0.1.0 +- kubesphere/examples-bookinfo-productpage-v1:1.16.2 +- kubesphere/examples-bookinfo-reviews-v1:1.16.2 +- kubesphere/examples-bookinfo-reviews-v2:1.16.2 +- kubesphere/examples-bookinfo-reviews-v3:1.16.2 +- kubesphere/examples-bookinfo-details-v1:1.16.2 +- kubesphere/examples-bookinfo-ratings-v1:1.16.3 +- busybox:1.31.1 +- joosthofman/wget:1.0 +- kubesphere/netshoot:v1.0 +- wordpress:4.8-apache +- mirrorgooglecontainers/hpa-example:latest +- java:openjdk-8-jre-alpine +- fluent/fluentd:v1.4.2-2.0 +- perl:latest +- osixia/openldap:1.3.0 +- redis:5.0.5-alpine +- alpine:3.10.4 +- haproxy:2.0.4 +- nginx:1.14-alpine +- minio/minio:RELEASE.2019-08-07T01-59-21Z +- minio/mc:RELEASE.2019-08-07T23-14-43Z +- kubesphere/elasticsearch-oss:6.7.0-1 +- kubesphere/s2irun:v2.1.1 +- kubesphere/builder-base:v3.1.0 +- kubesphere/builder-nodejs:v3.1.0 +- kubesphere/builder-maven:v3.1.0 +- kubesphere/builder-go:v3.1.0 +- kubesphere/s2i-binary:v2.1.0 +- kubesphere/tomcat85-java11-centos7:v2.1.0 +- kubesphere/tomcat85-java11-runtime:v2.1.0 +- kubesphere/tomcat85-java8-centos7:v2.1.0 +- kubesphere/tomcat85-java8-runtime:v2.1.0 +- kubesphere/java-11-centos7:v2.1.0 +- kubesphere/java-8-centos7:v2.1.0 +- kubesphere/java-8-runtime:v2.1.0 +- kubesphere/java-11-runtime:v2.1.0 diff --git a/ansible/make.yml b/ansible/make.yml index 87dd9fc1..c1dcb59b 100644 --- a/ansible/make.yml +++ b/ansible/make.yml @@ -4,7 +4,7 @@ vars: target_env: "{{ lookup('env', 'target') }}" roles: - - docker-1.0.8 + - docker-1.0.10 - docker-images - hosts: k8s-client,k8s-node,gpu-node @@ -33,8 +33,8 @@ vars: target_env: "{{ lookup('env', 'target') }}" roles: - - qingcloud-cli-1.0.6 - - docker-1.0.8 + - qingcloud-cli-1.0.7 + - docker-1.0.10 - etcd-1.1.0 - k8s-node # put extra modules above diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 3e00af03..0553952c 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -3,7 +3,7 @@ - src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.5.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.1.0.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/create-service-user-1.0.0.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/docker-1.0.8.tar.gz +- src: https://qingcloudappcenter.github.io/ansible-roles/docker-1.0.10.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/golang-1.0.3.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/etcd-1.1.0.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/install-1.0.6.tar.gz @@ -12,4 +12,4 @@ - src: https://qingcloudappcenter.github.io/ansible-roles/disable-apt-jobs-1.0.0.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/disable-motd-1.0.0.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/update-apt-sources-1.0.0.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/qingcloud-cli-1.0.6.tar.gz +- src: https://qingcloudappcenter.github.io/ansible-roles/qingcloud-cli-1.0.7.tar.gz diff --git a/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl b/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl index 0c229691..3e1d970a 100644 --- a/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl +++ b/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl @@ -22,6 +22,6 @@ LB_IP_FROM_V1={{ $lbIpFromV1 }} KS_MODULES_COUNT=10 {{- else }} {{- $extraModules := getvs "/env/extra.modules" }} -KS_MODULES_COUNT={{ add 2 (len (split (join $extraModules "") "," | filter "ks-*")) }} +KS_MODULES_COUNT={{ add 3 (len (split (join $extraModules "") "," | filter "ks-*")) }} {{- end }} NODE_ENV_EOF diff --git a/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh b/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh index 86e08c87..7609ea71 100644 --- a/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh +++ b/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh @@ -87,9 +87,11 @@ checkKsInstallerDone() { local output; output="$(runKubectl -n kubesphere-system logs --tail 50 $podName)" || return $EC_KS_INSTALL_LOGS_ERR if echo "$output" | grep "^PLAY RECAP **" -A1 | egrep -o "failed=[1-9]"; then return $EC_KS_INSTALL_FAILED; fi echo "$output" | grep -oF 'Welcome to KubeSphere!' || return $EC_KS_INSTALL_RUNNING - local endStrings="total: $KS_MODULES_COUNT completed:$KS_MODULES_COUNT" + #local endStrings="is successful ($KS_MODULES_COUNT/$KS_MODULES_COUNT)" if $IS_UPGRADING_FROM_V2; then endStrings=" failed=0 "; fi - echo "$output" | grep "Welcome to KubeSphere!" -B4 | grep -oF "$endStrings" || return $EC_KS_INSTALL_DONE_WITH_ERR + # if tail of installer log has line like "task openpitrix status is failed", means one or more components are failed + # to install. + !(echo "$output" | grep "Welcome to KubeSphere!" -B30 | grep -q "^task.*failed") || return $EC_KS_INSTALL_DONE_WITH_ERR } getKsInstallerPodName() { diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl index 6a262b81..a8419aea 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl @@ -44,6 +44,7 @@ ALL_NODES_EOF myRole={{ getv "/host/role" }} mySid={{ getv "/host/sid" }} myInstanceId={{ getv "/host/instance_id" }} +myZone={{ getv "/host/zone" }} myNodeName=$(buildNodeName $myRole $mySid) flush /opt/app/current/bin/envs/node.env << NODE_ENV_EOF @@ -53,10 +54,12 @@ CLUSTER_TAG={{ join (getvs "/cluster/cluster_tag") "" }} CLUSTER_VXNET={{ getv "/cluster/vxnet" }} CLUSTER_ZONE={{ getv "/cluster/zone" }} CLUSTER_API_SERVER={{ getv "/cluster/api_server/host" "api.qingcloud.com" }} +{{- if exists "/cluster/region_id" }}CLUSTER_REGION={{ getv "/cluster/region_id" }}{{- end }} MY_IP={{ getv "/host/ip" }} MY_SID=$mySid MY_ROLE=$myRole MY_NODE_NAME=$myNodeName +MY_ZONE=$myZone MY_INSTANCE_ID=$myInstanceId UPGRADED_FROM_V1={{ $upgradedFromV1 }} LB_IP_FROM_V1={{ $lbIpFromV1 }} diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl index 381f0f8e..d5ec4c23 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl @@ -34,7 +34,9 @@ apiServer: advertise-address: 0.0.0.0 {{- if $kubeAuditPolicy }} audit-policy-file: $KUBE_AUDIT_POLICY_RUNTIME_FILE + {{- if $kubeAuditWebhook }} audit-webhook-config-file: $KUBE_AUDIT_WEBHOOK_RUNTIME_FILE + {{- end }} audit-log-maxage: "{{ getv "/env/keep_audit_days" }}" audit-log-path: /etc/kubernetes/audit/logs/audit.log audit-log-maxsize: "{{ getv "/env/kube_audit_log_maxsize" "1" }}" diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl index 6cf122be..f93f179d 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl @@ -28,7 +28,8 @@ common: logging: enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "logging")) 0) }} openpitrix: - enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "openpitrix")) 0) }} + store: + enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "openpitrix")) 0) }} devops: enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "devops")) 0) }} servicemesh: diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh index 51588160..921c3adc 100644 --- a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh +++ b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh @@ -73,6 +73,7 @@ initCluster() { warmUpLocalDns if isFirstMaster; then initFirstNode; else initOtherNode; fi annotateInstanceId + labelTopology rm -rf $JOIN_CMD_FILE log --debug "done initializing cluster!" } @@ -196,7 +197,7 @@ upgrade() { check() { _check - checkNodeStats '$2~/^Ready/' $MY_NODE_NAME + checkNodeStats '$2~/^Ready/' $(getMyNodeName) } checkSvc() { @@ -212,11 +213,14 @@ revive() { _revive } +# in v1.19.8 metrics are changed from: +# kubelet_running_container_count => kubelet_running_containers +# kubelet_running_pod_count => kubelet_running_pods measure() { isClusterInitialized && isNodeInitialized || return 0 local -r regex="$(sed 's/^\s*//g' <<< ' - kubelet_running_container_count{container_state="running"} - kubelet_running_pod_count + kubelet_running_containers{container_state="running"} + kubelet_running_pods ' | paste -sd'|' | sed 's/^|/^(/; s/|$/)/')" runKubectl get -s https://localhost:10250 --raw /metrics --insecure-skip-tls-verify | egrep "$regex" | sed -r 's/\{[^}]+\}//g; s/ /: /g' | yq -j r - } @@ -554,6 +558,11 @@ setUpNodeLocalDns() { sed "$replaceRules" /opt/app/current/conf/k8s/nodelocaldns-$K8S_VERSION.yml | runKubectl apply -f - } +countUnBoundPVCs() { + count=$(runKubectl get pvc -A --no-headers | grep -v Bound | wc -l) + return ${count} +} + _setUpStorage() { # remove previous version if $IS_UPGRADING_FROM_V2; then @@ -563,6 +572,15 @@ _setUpStorage() { # CSI plugin local -r csiChartFile=/opt/app/current/conf/k8s/csi-qingcloud-$QINGCLOUD_CSI_VERSION.tgz local -r csiValuesFile=/opt/app/current/conf/k8s/csi-qingcloud-values.yml + + # Need to uninstall and reinstall if upgrading, because helm upgrade will fail due to + # immutable fields change during upgrade. + if $IS_UPGRADING; then + # make sure there no pending pvs, if not skip upgrading csi-qingcloud + retry 600 1 0 countUnBoundPVCs || return 0 + runHelm -n kube-system uninstall csi-qingcloud + fi + yq p $QINGCLOUD_CONFIG config | cat - $csiValuesFile | \ runHelm -n kube-system upgrade --install csi-qingcloud $csiChartFile -f - @@ -627,7 +645,12 @@ launchKs() { buildKsDynamicConf() { local -r ksCfgDynamicFile=/opt/app/current/conf/k8s/ks-config.dynamic.yml - yq p $ksCfgDynamicFile spec + if $IS_UPGRADING; then + # components could be manually enabled + runKubectl -n kubesphere-system get cc ks-installer -o yaml | yq r - 'spec' | yq p - spec + else + yq p $ksCfgDynamicFile spec + fi } buildKsConf() { @@ -890,6 +913,13 @@ annotateInstanceId() { runKubectl annotate no $(getMyNodeName) node.beta.kubernetes.io/instance-id="$MY_INSTANCE_ID" } +labelTopology() { + runKubectl label no $(getMyNodeName) topology.kubernetes.io/zone="$MY_ZONE" --overwrite + if [ ! -z "${CLUSTER_REGION}" ]; then + runKubectl label no $(getMyNodeName) topology.kubernetes.io/region="$CLUSTER_REGION" --overwrite + fi +} + markAllInOne() { local hostName; hostName=${1:-$(getMyNodeName)} runKubectl taint node $hostName node-role.kubernetes.io/master:NoSchedule- diff --git a/ansible/roles/app-role-k8s/tasks/main.yml b/ansible/roles/app-role-k8s/tasks/main.yml index 7bb76c18..34c9eada 100644 --- a/ansible/roles/app-role-k8s/tasks/main.yml +++ b/ansible/roles/app-role-k8s/tasks/main.yml @@ -55,10 +55,10 @@ url: https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/{{ nvidia_plugin_version }}/nvidia-device-plugin.yml - name: hostnic version: "{{ hostnic_version }}" - url: https://raw.githubusercontent.com/yunify/hostnic-cni/{{ hostnic_version }}/deploy/hostnic.yaml + url: https://raw.githubusercontent.com/yunify/hostnic-cni/v{{ hostnic_version }}/deploy/hostnic.yaml - name: hostnic-policy version: "{{ hostnic_version }}" - url: https://raw.githubusercontent.com/yunify/hostnic-cni/{{ hostnic_version }}/policy/calico.yaml + url: https://raw.githubusercontent.com/yunify/hostnic-cni/v{{ hostnic_version }}/policy/calico.yaml loop_control: loop_var: pkg diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index e8c2f2ed..799dcd68 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -12,13 +12,12 @@ "advanced_actions": ["scale_horizontal"], "unsupported_actions": ["rollback"], "upgrade_policy": [ - "appv-egh21tjg", - "appv-39iiioy3" + "appv-egh21tjg" ], "upgrading_policy": "in-place-parallel", "in-place-upgrade-nodes": [{ "container":{ - "snapshot": "ss-z9w1oyol", + "snapshot": "ss-pned1k8r", "zone": "pek3" }, "copy":[{ @@ -44,7 +43,7 @@ }] }, { "container":{ - "snapshot": "ss-n50f95x6", + "snapshot": "ss-mixs6spp", "zone": "pek3" }, "copy":[{ @@ -58,7 +57,7 @@ "role": "master", "container": { "type": "kvm", - "image": "img-gtc2f1o7", + "image": "img-sve8d2fj", "zone": "pek3" }, "instance_class": {{cluster.master.instance_class}}, @@ -144,7 +143,7 @@ "role": "node_perf", "container": { "type": "kvm", - "image": "img-gtc2f1o7", + "image": "img-sve8d2fj", "zone": "pek3" }, "instance_class": {{cluster.node_perf.instance_class}}, @@ -206,7 +205,7 @@ "role": "node_super_perf", "container": { "type": "kvm", - "image": "img-gtc2f1o7", + "image": "img-sve8d2fj", "zone": "pek3" }, "instance_class": {{cluster.node_super_perf.instance_class}}, @@ -268,7 +267,7 @@ "role": "node_gpu", "container": { "type": "kvm", - "image": "img-2tsq1fsz", + "image": "img-kirp1glw", "zone": "pek3" }, "instance_class": {{cluster.node_gpu.instance_class}}, @@ -329,7 +328,7 @@ "role": "client", "container": { "type": "kvm", - "image": "img-2gfk2szm", + "image": "img-swxveyde", "zone": "pek3" }, "instance_class": {{cluster.client.instance_class}}, @@ -382,13 +381,13 @@ "enable": true, "cmd": "appctl measure", "items": { - "kubelet_running_container_count": { + "kubelet_running_containers": { "unit": "", "value_type": "int", "statistics_type": "latest", "scale_factor_when_display": 1 }, - "kubelet_running_pod_count": { + "kubelet_running_pods": { "unit": "", "value_type": "int", "statistics_type": "latest", @@ -396,10 +395,10 @@ } }, "groups": { - "kubelet": ["kubelet_running_pod_count", "kubelet_running_container_count"] + "kubelet": ["kubelet_running_pods", "kubelet_running_containers"] }, "display": ["kubelet"], - "alarm": ["kubelet_running_container_count"] + "alarm": ["kubelet_running_containers"] }, "endpoints": { "nodeport": { diff --git a/app/config.json b/app/config.json index 166d1572..4f6f73c5 100644 --- a/app/config.json +++ b/app/config.json @@ -21,13 +21,14 @@ }, { "key": "resource_group", "label": "Resource Configuration", - "description": "The resource configuration of the service. Single master cluster cannot upgrade to HA mster cluster.", + "description": "Resource Group Description", "disable_when_scale": true, "type": "string", "default": "high-performance test", "range": [ "standard dev", "high-performance test", + "standard prod", "high-performance prod" ] }, { @@ -78,7 +79,7 @@ "range": [101, 202], "default": 202, "required": "yes", - "resource_group": [101, 202, 202] + "resource_group": [101, 202, 101, 202] }, { "key": "count", "label": "count", @@ -88,7 +89,7 @@ "range": [1, 3], "required": "yes", "changeable": false, - "resource_group": [1, 1, 3] + "resource_group": [1, 1, 3, 3] }, { "key": "cpu_model", "label": "CPU Model", @@ -105,7 +106,7 @@ "default": 8, "range": [4, 8, 12, 16, 24, 32, 64], "required": "yes", - "resource_group": [4, 4, 8] + "resource_group": [4, 4, 8, 8] }, { "key": "memory", "label": "Memory", @@ -114,7 +115,7 @@ "default": 16384, "range": [8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, 131072, 196608, 262144], "required": "yes", - "resource_group": [8192, 8192, 16384] + "resource_group": [8192, 8192, 16384, 16384] }, { "key": "volume_size", "label": "volume size", @@ -150,7 +151,7 @@ "min": 0, "max": 100, "required": "yes", - "resource_group": [2, 0, 0] + "resource_group": [2, 0, 3, 0] }, { "key": "cpu_model", "label": "CPU Model", @@ -210,7 +211,7 @@ "min": 0, "max": 100, "required": "yes", - "resource_group": [0, 2, 3] + "resource_group": [0, 2, 0, 3] }, { "key": "cpu_model", "label": "CPU Model", @@ -268,7 +269,7 @@ "min": 0, "max": 100, "default": 0, - "resource_group": [0, 0, 0], + "resource_group": [0, 0, 0, 0], "auto_scale_step": 1, "required": "yes" }, { @@ -337,7 +338,7 @@ "range": [101, 202], "default": 202, "required": "yes", - "resource_group": [101, 202, 202] + "resource_group": [101, 202, 101, 202] }, { "key": "cpu", "label": "CPU", @@ -346,7 +347,7 @@ "default": 2, "range": [1, 2], "required": "yes", - "resource_group": [1, 2, 2] + "resource_group": [1, 2, 1, 2] }, { "key": "memory", "label": "Memory", @@ -355,7 +356,7 @@ "default": 4096, "range": [1024, 2048, 4096], "required": "yes", - "resource_group": [2048, 4096, 4096] + "resource_group": [2048, 4096, 2048, 4096] }, { "key": "count", "label": "count", @@ -400,7 +401,6 @@ "ks-openpitrix", "ks-devops", "ks-servicemesh", - "ks-notification", "ks-alerting" ], "default": "metrics-server", diff --git a/app/locale/en.json b/app/locale/en.json index 07c036f5..b1f33329 100644 --- a/app/locale/en.json +++ b/app/locale/en.json @@ -3,5 +3,6 @@ "KS is not installed.": "KubeSphere is not installed.\nTo install KubeSphere, you can refer to [docs](https://docs.qingcloud.com/product/container/qke/), or [submit a ticket](https://console.qingcloud.com/tickets/) for support.", "Using master node IP. Please try again later when external IP is ready.": "The LB with EIP is being created, please refresh this tab later; currently the IP of the first control plane is displayed; if this message appears for a long time, please check the ks-console service under kubesphere-system namespace", "API access key id": "QingCloud IaaS [API Access Key](https://console.qingcloud.com/access_keys/), which will be used to create QingCloud resources, such as load balancers, volumes, etc.", - "Whether to install kubesphere": "Whether to install KubeSphere, KubeSphere and monitoring components will be installed by default;To install KubeSphere, you can refer to [docs](https://docs.qingcloud.com/product/container/qke/), or [submit a ticket](https://console.qingcloud.com/tickets/) for support." + "Whether to install kubesphere": "Whether to install KubeSphere, KubeSphere and monitoring components will be installed by default;To install KubeSphere, you can refer to [docs](https://docs.qingcloud.com/product/container/qke/), or [submit a ticket](https://console.qingcloud.com/tickets/) for support.", + "Resource Group Description": "The resource configuration of the service. Single master cluster cannot upgrade to HA mster cluster." } diff --git a/app/locale/zh-cn.json b/app/locale/zh-cn.json index f74b5a51..35bab189 100644 --- a/app/locale/zh-cn.json +++ b/app/locale/zh-cn.json @@ -85,6 +85,8 @@ "parameters for kubelet": "kubelet 参数，自定义配置，支持多项配置，需严格遵循每行配置一项且保持 `key=value` 的格式，配置示例：`--add-dir-header=true`，默认值 `--eviction-hard=memory.available<5%` 表示当节点剩余内存不足 5% 时 kubelet 会立即关掉选中的容器组来释放内存，`--eviction-soft=memory.available<10%` 与 `--eviction-soft-grace-period=memory.available=2m` 表示当可用内存连续 2 分钟不足 10% 时，会平滑关闭（graceful shutdown）选中的容器组；注意 `--eviction-soft` 与 `--eviction-soft-grace-period` 必须同时指定，否则 kubelet 将无法启动；其他配置项请参考官方文档 [kubelet configurations](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/)，使用时请保留 `--` 符号", "kubelet_running_container_count": "正在运行的容器数量", "kubelet_running_pod_count": "正在运行的 Pod 数量", + "kubelet_running_containers": "正在运行的容器数量", + "kubelet_running_pods": "正在运行的 Pod 数量", "Fluent forward server": "Fluent 日志转发服务", "The fluent log server address to forward server, format host:port": "Fluent 日志服务器，用于将 Kubernetes 收集到的日志转发到用户自定义的日志服务，格式 host:port", "The Docker hub registry mirrors, use a blank to split multi registry mirrors": "完整的 Docker 镜像服务地址，比如 https://mirror.harbor.local；多个地址之间用空格隔开", @@ -127,8 +129,9 @@ "Resource Configuration": "快速配置", "standard dev": "基础型开发环境", "high-performance test": "企业型测试环境", + "standard prod": "基础性生产环境", "high-performance prod": "企业型生产环境", - "The resource configuration of the service. Single master cluster cannot upgrade to HA mster cluster.": "请选择合适的预制资源配置类型，快速定义集群配置。也可根据自身需求自定义节点配置。非高可用集群不可以升级到高可用集群", + "Resource Group Description": "请选择合适的预制资源配置类型，快速定义集群配置。也可根据自身需求自定义节点配置。非高可用集群不可以升级到高可用集群", "host aliases": "主机 hosts 记录", "Set host aliases": "自定义添加到 /etc/hosts 文件的记录，比如 '192.168.2.2 host1,192.168.2.3 host2'，多条记录用逗号分割", "The insecure Docker registry, use a blank to split multi registry": "需要通过非安全的 HTTP 或不受信任的 HTTPS 访问的 Docker 仓库，比如 mirror.harbor.local，多个地址通过空格切分",