From a1ac74d3224b555a54eb019afa137c2faa5c4c55 Mon Sep 17 00:00:00 2001 From: Hongliang Wang Date: Wed, 12 May 2021 19:27:43 +0800 Subject: [PATCH 01/13] bump kubesphere to v3.1 Signed-off-by: Jeff --- ansible/group_vars/all.yml | 135 +++++++++++------- ansible/make.yml | 2 +- ansible/requirements.yml | 2 +- .../templates/client.sh/01.node.env.tmpl | 2 +- .../files/opt/app/current/bin/node/client.sh | 2 +- .../kube.sh/09.ks-config.dynamic.yml.tmpl | 3 +- ansible/roles/app-role-k8s/tasks/main.yml | 4 +- app/cluster.json.mustache | 14 +- app/config.json | 1 - 9 files changed, 98 insertions(+), 67 deletions(-) diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 526aab2a..39a7590f 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -1,25 +1,24 @@ -app_version: 3.0.1 +app_version: 3.1.0 gcr_mirror: kubesphere -etcd_version: 3.3.12 +etcd_version: 3.4.13 cni_version: 0.8.6 -k8s_version: 1.18.12 +k8s_version: 1.19.8 flannel_version: 0.12.0 coredns_version: 1.6.9 -calico_version: 3.15.1 -hostnic_version: 1.0.0 +calico_version: 3.16.3 +hostnic_version: 1.0.0-alpha4 qingcloud_csi_version: 1.2.6 qingcloud_ccm_version: 1.4.5 -ks_version: 3.0.0 -ks_installer_image_tag: qke-3.0.1 +ks_version: 3.1.0 +ks_installer_image_tag: v3.1.0 helm_version: 3.2.1 helm_2to3_version: 0.8.0 helm_stable_repo: https://charts.kubesphere.io/mirror helm_main_repo: https://charts.kubesphere.io/main helm_test_repo: https://charts.kubesphere.io/test -nodelocaldns_version: 1.15.5 +nodelocaldns_version: 1.15.12 nvidia_tesla_version: 418.116.00 nvidia_plugin_version: 1.0.0-beta4 -# 本地 images package 的位置 local_images_path: /images ks_images_package: allks.tgz k8s_images_package: allk8s.tgz @@ -34,46 +33,6 @@ binaries: docker_images_k8s: - kubesphere/cloud-controller-manager:v1.4.4 - -docker_images_ks: -- alpine:3.10.4 -- docker:19.03 -- haproxy:2.0.4 -- jenkins/jenkins:2.176.2 -- kubesphere/builder-base:v2.1.0 -- kubesphere/builder-go:v2.1.0 -- kubesphere/builder-maven:v2.1.0 -- kubesphere/builder-nodejs:v2.1.0 -- kubesphere/elasticsearch-oss:6.7.0-1 -- kubesphere/etcd:v3.2.18 -- kubesphere/java-11-centos7:v2.1.0 -- kubesphere/java-11-runtime:v2.1.0 -- kubesphere/java-8-centos7:v2.1.0 -- kubesphere/java-8-runtime:v2.1.0 -- kubesphere/kubectl:v1.0.0 -- kubesphere/kube-rbac-proxy:v0.4.1 -- kubesphere/nodejs-4-centos7:v2.1.0 -- kubesphere/nodejs-6-centos7:v2.1.0 -- kubesphere/nodejs-8-centos7:v2.1.0 -- kubesphere/python-27-centos7:v2.1.0 -- kubesphere/python-34-centos7:v2.1.0 -- kubesphere/python-35-centos7:v2.1.0 -- kubesphere/python-36-centos7:v2.1.0 -- kubesphere/s2i-binary:v2.1.0 -- kubesphere/s2ioperator:v2.1.1 -- kubesphere/s2irun:v2.1.1 -- kubesphere/tomcat85-java11-centos7:v2.1.0 -- kubesphere/tomcat85-java11-runtime:v2.1.0 -- kubesphere/tomcat85-java8-centos7:v2.1.0 -- kubesphere/tomcat85-java8-runtime:v2.1.0 -- minio/mc:RELEASE.2019-08-07T23-14-43Z -- minio/minio:RELEASE.2019-08-07T01-59-21Z -- mysql:8.0.11 -- nginx:1.14-alpine -- osixia/openldap:1.3.0 -- redis:5.0.5-alpine - -docker_images_k8s_new: - calico/cni:v3.15.1 - calico/kube-controllers:v3.15.1 - calico/node:v3.15.1 @@ -86,7 +45,7 @@ docker_images_k8s_new: - csiplugin/csi-resizer:v0.4.0 - csiplugin/csi-snapshotter:v2.0.1 - csiplugin/snapshot-controller:v2.0.1 -- kubesphere/hostnic:v1.0.0 +- kubesphere/hostnic:v1.0.0-alpha4 - nvidia/k8s-device-plugin:1.0.0-beta4 - kubesphere/flannel:v0.12.0 - kubesphere/kube-apiserver:v1.18.12 @@ -95,7 +54,7 @@ docker_images_k8s_new: - kubesphere/kube-scheduler:v1.18.12 - kubesphere/pause:3.2 -docker_images_ks_new: +docker_images_ks: - istio/citadel:1.4.8 - istio/galley:1.4.8 - istio/kubectl:1.4.8 @@ -121,7 +80,7 @@ docker_images_ks_new: - kubesphere/ks-console:v3.0.0 - kubesphere/ks-controller-manager:v3.0.0 - kubesphere/ks-devops:flyway-v3.0.0 -- kubesphere/ks-installer:qke-3.0.0 +- kubesphere/ks-installer:v3.1.0 - kubesphere/ks-upgrade:v3.0.0 - kubesphere/kube-auditing-operator:v0.1.2 - kubesphere/kube-auditing-webhook:v0.1.2 @@ -149,3 +108,75 @@ docker_images_ks_new: - openpitrix/release-app:sha-303629d - prom/alertmanager:v0.21.0 - prom/prometheus:v2.20.1 + +docker_images_k8s_new: +- kubesphere/kube-apiserver:v1.19.8 +- kubesphere/kube-scheduler:v1.19.8 +- kubesphere/kube-proxy:v1.19.8 +- kubesphere/kube-controller-manager:v1.19.8 +- kubesphere/pause:3.1 +- kubesphere/etcd:v3.4.13 +- calico/cni:v3.16.3 +- calico/kube-controllers:v3.16.3 +- calico/node:v3.16.3 +- calico/pod2daemon-flexvol:v3.16.3 +- kubesphere/k8s-dns-node-cache:1.15.12 +- kubesphere/nfs-client-provisioner:v3.1.0-k8s1.11 +- csiplugin/csi-neonsan:v1.2.0 +- csiplugin/csi-neonsan-ubuntu:v1.2.0 +- csiplugin/csi-neonsan-centos:v1.2.0 + +docker_images_ks_new: +- openebs/provisioner-localpv:2.3.0 +- openebs/linux-utils:2.3.0 +- kubesphere/ks-apiserver:v3.1.0 +- kubesphere/ks-console:v3.1.0 +- kubesphere/ks-controller-manager:v3.1.0 +- kubesphere/ks-installer:v3.1.0 +- kubesphere/kubectl:v1.19.0 +- kubesphere/nginx-ingress-controller:v0.35.0 +- kubesphere/kubefed:v0.7.0 +- kubesphere/tower:v0.2.0 +- kubesphere/prometheus-config-reloader:v0.42.1 +- kubesphere/prometheus-operator:v0.42.1 +- prom/prometheus:v2.26.0 +- prom/node-exporter:v0.18.1 +- kubesphere/ks-alerting-migration:v3.1.0 +- kubesphere/notification-manager-operator:v1.0.0 +- kubesphere/notification-manager:v1.0.0 +- kubesphere/metrics-server:v0.4.2 +- kubesphere/kube-rbac-proxy:v0.8.0 +- kubesphere/kube-state-metrics:v1.9.7 +- openebs/provisioner-localpv:2.3.0 +- thanosio/thanos:v0.18.0 +- grafana/grafana:7.4.3 +- kubesphere/fluentbit-operator:v0.5.0 +- kubesphere/fluent-bit:v1.6.9 +- elastic/filebeat:6.7.0 +- kubesphere/kube-events-ruler:v0.2.0 +- istio/pilot:1.6.10 +- istio/proxyv2:1.6.10 +- kubesphere/kiali:v1.26.1 +- kubesphere/kiali-operator:v1.26.1 +- kubesphere/ks-jenkins:2.249.1 +- kubesphere/s2ioperator:v3.1.0 +- kubesphere/openpitrix-jobs:v3.1.0 +- weaveworks/scope:1.13.0 +- kubeedge/cloudcore:v1.6.1 +- kubesphere/edge-watcher:v0.1.0 +- kubesphere/kube-rbac-proxy:v0.5.0 +- kubesphere/edge-watcher-agent:v0.1.0 +- kubesphere/examples-bookinfo-productpage-v1:1.16.2 +- kubesphere/examples-bookinfo-reviews-v1:1.16.2 +- kubesphere/examples-bookinfo-reviews-v2:1.16.2 +- kubesphere/examples-bookinfo-reviews-v3:1.16.2 +- kubesphere/examples-bookinfo-details-v1:1.16.2 +- kubesphere/examples-bookinfo-ratings-v1:1.16.3 +- busybox:1.31.1 +- joosthofman/wget:1.0 +- kubesphere/netshoot:v1.0 +- wordpress:4.8-apache +- mirrorgooglecontainers/hpa-example:latest +- java:openjdk-8-jre-alpine +- fluent/fluentd:v1.4.2-2.0 +- perl:latest \ No newline at end of file diff --git a/ansible/make.yml b/ansible/make.yml index 87dd9fc1..3fd336fb 100644 --- a/ansible/make.yml +++ b/ansible/make.yml @@ -33,7 +33,7 @@ vars: target_env: "{{ lookup('env', 'target') }}" roles: - - qingcloud-cli-1.0.6 + - qingcloud-cli-1.0.7 - docker-1.0.8 - etcd-1.1.0 - k8s-node diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 3e00af03..f173f900 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -12,4 +12,4 @@ - src: https://qingcloudappcenter.github.io/ansible-roles/disable-apt-jobs-1.0.0.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/disable-motd-1.0.0.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/update-apt-sources-1.0.0.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/qingcloud-cli-1.0.6.tar.gz +- src: https://qingcloudappcenter.github.io/ansible-roles/qingcloud-cli-1.0.7.tar.gz diff --git a/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl b/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl index 0c229691..3e1d970a 100644 --- a/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl +++ b/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl @@ -22,6 +22,6 @@ LB_IP_FROM_V1={{ $lbIpFromV1 }} KS_MODULES_COUNT=10 {{- else }} {{- $extraModules := getvs "/env/extra.modules" }} -KS_MODULES_COUNT={{ add 2 (len (split (join $extraModules "") "," | filter "ks-*")) }} +KS_MODULES_COUNT={{ add 3 (len (split (join $extraModules "") "," | filter "ks-*")) }} {{- end }} NODE_ENV_EOF diff --git a/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh b/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh index 86e08c87..a8934e82 100644 --- a/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh +++ b/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh @@ -87,7 +87,7 @@ checkKsInstallerDone() { local output; output="$(runKubectl -n kubesphere-system logs --tail 50 $podName)" || return $EC_KS_INSTALL_LOGS_ERR if echo "$output" | grep "^PLAY RECAP **" -A1 | egrep -o "failed=[1-9]"; then return $EC_KS_INSTALL_FAILED; fi echo "$output" | grep -oF 'Welcome to KubeSphere!' || return $EC_KS_INSTALL_RUNNING - local endStrings="total: $KS_MODULES_COUNT completed:$KS_MODULES_COUNT" + local endStrings="is successful ($KS_MODULES_COUNT/$KS_MODULES_COUNT)" if $IS_UPGRADING_FROM_V2; then endStrings=" failed=0 "; fi echo "$output" | grep "Welcome to KubeSphere!" -B4 | grep -oF "$endStrings" || return $EC_KS_INSTALL_DONE_WITH_ERR } diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl index 6cf122be..f93f179d 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl @@ -28,7 +28,8 @@ common: logging: enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "logging")) 0) }} openpitrix: - enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "openpitrix")) 0) }} + store: + enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "openpitrix")) 0) }} devops: enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "devops")) 0) }} servicemesh: diff --git a/ansible/roles/app-role-k8s/tasks/main.yml b/ansible/roles/app-role-k8s/tasks/main.yml index 7bb76c18..34c9eada 100644 --- a/ansible/roles/app-role-k8s/tasks/main.yml +++ b/ansible/roles/app-role-k8s/tasks/main.yml @@ -55,10 +55,10 @@ url: https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/{{ nvidia_plugin_version }}/nvidia-device-plugin.yml - name: hostnic version: "{{ hostnic_version }}" - url: https://raw.githubusercontent.com/yunify/hostnic-cni/{{ hostnic_version }}/deploy/hostnic.yaml + url: https://raw.githubusercontent.com/yunify/hostnic-cni/v{{ hostnic_version }}/deploy/hostnic.yaml - name: hostnic-policy version: "{{ hostnic_version }}" - url: https://raw.githubusercontent.com/yunify/hostnic-cni/{{ hostnic_version }}/policy/calico.yaml + url: https://raw.githubusercontent.com/yunify/hostnic-cni/v{{ hostnic_version }}/policy/calico.yaml loop_control: loop_var: pkg diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index e8c2f2ed..9f6b254c 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -18,7 +18,7 @@ "upgrading_policy": "in-place-parallel", "in-place-upgrade-nodes": [{ "container":{ - "snapshot": "ss-z9w1oyol", + "snapshot": "ss-8ss6qpy1", "zone": "pek3" }, "copy":[{ @@ -44,7 +44,7 @@ }] }, { "container":{ - "snapshot": "ss-n50f95x6", + "snapshot": "ss-s85qohr2", "zone": "pek3" }, "copy":[{ @@ -58,7 +58,7 @@ "role": "master", "container": { "type": "kvm", - "image": "img-gtc2f1o7", + "image": "img-y3b5ut5q", "zone": "pek3" }, "instance_class": {{cluster.master.instance_class}}, @@ -144,7 +144,7 @@ "role": "node_perf", "container": { "type": "kvm", - "image": "img-gtc2f1o7", + "image": "img-y3b5ut5q", "zone": "pek3" }, "instance_class": {{cluster.node_perf.instance_class}}, @@ -206,7 +206,7 @@ "role": "node_super_perf", "container": { "type": "kvm", - "image": "img-gtc2f1o7", + "image": "img-y3b5ut5q", "zone": "pek3" }, "instance_class": {{cluster.node_super_perf.instance_class}}, @@ -268,7 +268,7 @@ "role": "node_gpu", "container": { "type": "kvm", - "image": "img-2tsq1fsz", + "image": "img-9h8jlw6j", "zone": "pek3" }, "instance_class": {{cluster.node_gpu.instance_class}}, @@ -329,7 +329,7 @@ "role": "client", "container": { "type": "kvm", - "image": "img-2gfk2szm", + "image": "img-5jbi2wg2", "zone": "pek3" }, "instance_class": {{cluster.client.instance_class}}, diff --git a/app/config.json b/app/config.json index 166d1572..fda08900 100644 --- a/app/config.json +++ b/app/config.json @@ -400,7 +400,6 @@ "ks-openpitrix", "ks-devops", "ks-servicemesh", - "ks-notification", "ks-alerting" ], "default": "metrics-server", From 7d8a98881917b7d0158100aa0a1e6fa58b953a67 Mon Sep 17 00:00:00 2001 From: Jeff Date: Thu, 13 May 2021 18:17:45 +0800 Subject: [PATCH 02/13] upgrade hostnic version to v1.0.0-alpha.5 Signed-off-by: Jeff --- ansible/group_vars/all.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 39a7590f..0d20a845 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -6,7 +6,7 @@ k8s_version: 1.19.8 flannel_version: 0.12.0 coredns_version: 1.6.9 calico_version: 3.16.3 -hostnic_version: 1.0.0-alpha4 +hostnic_version: 1.0.0-alpha.5 qingcloud_csi_version: 1.2.6 qingcloud_ccm_version: 1.4.5 ks_version: 3.1.0 @@ -45,7 +45,7 @@ docker_images_k8s: - csiplugin/csi-resizer:v0.4.0 - csiplugin/csi-snapshotter:v2.0.1 - csiplugin/snapshot-controller:v2.0.1 -- kubesphere/hostnic:v1.0.0-alpha4 +- kubesphere/hostnic:v1.0.0-alpha.5 - nvidia/k8s-device-plugin:1.0.0-beta4 - kubesphere/flannel:v0.12.0 - kubesphere/kube-apiserver:v1.18.12 @@ -179,4 +179,4 @@ docker_images_ks_new: - mirrorgooglecontainers/hpa-example:latest - java:openjdk-8-jre-alpine - fluent/fluentd:v1.4.2-2.0 -- perl:latest \ No newline at end of file +- perl:latest From 947983c76c875e301290baf83293c59507f221fe Mon Sep 17 00:00:00 2001 From: Jeff Date: Thu, 13 May 2021 21:42:13 +0800 Subject: [PATCH 03/13] fix kubesphere components status check script Signed-off-by: Jeff --- .../files/opt/app/current/bin/node/client.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh b/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh index a8934e82..7609ea71 100644 --- a/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh +++ b/ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh @@ -87,9 +87,11 @@ checkKsInstallerDone() { local output; output="$(runKubectl -n kubesphere-system logs --tail 50 $podName)" || return $EC_KS_INSTALL_LOGS_ERR if echo "$output" | grep "^PLAY RECAP **" -A1 | egrep -o "failed=[1-9]"; then return $EC_KS_INSTALL_FAILED; fi echo "$output" | grep -oF 'Welcome to KubeSphere!' || return $EC_KS_INSTALL_RUNNING - local endStrings="is successful ($KS_MODULES_COUNT/$KS_MODULES_COUNT)" + #local endStrings="is successful ($KS_MODULES_COUNT/$KS_MODULES_COUNT)" if $IS_UPGRADING_FROM_V2; then endStrings=" failed=0 "; fi - echo "$output" | grep "Welcome to KubeSphere!" -B4 | grep -oF "$endStrings" || return $EC_KS_INSTALL_DONE_WITH_ERR + # if tail of installer log has line like "task openpitrix status is failed", means one or more components are failed + # to install. + !(echo "$output" | grep "Welcome to KubeSphere!" -B30 | grep -q "^task.*failed") || return $EC_KS_INSTALL_DONE_WITH_ERR } getKsInstallerPodName() { From 53028377d3709e21a01cf5066ccf05d4fbfa08ff Mon Sep 17 00:00:00 2001 From: Jeff Date: Tue, 18 May 2021 19:20:40 +0800 Subject: [PATCH 04/13] upgrade csi version & fix missing monitoring stats Signed-off-by: Jeff --- ansible/group_vars/all.yml | 14 +++++++------ .../files/opt/app/current/bin/node/k8s-ctl.sh | 7 +++++-- app/cluster.json.mustache | 20 +++++++++---------- app/locale/zh-cn.json | 2 ++ 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 0d20a845..94008fcb 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -7,8 +7,8 @@ flannel_version: 0.12.0 coredns_version: 1.6.9 calico_version: 3.16.3 hostnic_version: 1.0.0-alpha.5 -qingcloud_csi_version: 1.2.6 -qingcloud_ccm_version: 1.4.5 +qingcloud_csi_version: 1.2.7 +qingcloud_ccm_version: 1.4.6 ks_version: 3.1.0 ks_installer_image_tag: v3.1.0 helm_version: 3.2.1 @@ -17,7 +17,7 @@ helm_stable_repo: https://charts.kubesphere.io/mirror helm_main_repo: https://charts.kubesphere.io/main helm_test_repo: https://charts.kubesphere.io/test nodelocaldns_version: 1.15.12 -nvidia_tesla_version: 418.116.00 +nvidia_tesla_version: 460.73.01 nvidia_plugin_version: 1.0.0-beta4 local_images_path: /images ks_images_package: allks.tgz @@ -122,9 +122,11 @@ docker_images_k8s_new: - calico/pod2daemon-flexvol:v3.16.3 - kubesphere/k8s-dns-node-cache:1.15.12 - kubesphere/nfs-client-provisioner:v3.1.0-k8s1.11 -- csiplugin/csi-neonsan:v1.2.0 -- csiplugin/csi-neonsan-ubuntu:v1.2.0 -- csiplugin/csi-neonsan-centos:v1.2.0 +- csiplugin/csi-qingcloud:v1.2.1 +- csiplugin/csi-neonsan:v1.2.1 +- csiplugin/csi-neonsan-ubuntu:v1.2.1 +- csiplugin/csi-neonsan-centos:v1.2.1 +- kubesphere/cloud-controller-manager:v1.4.6 docker_images_ks_new: - openebs/provisioner-localpv:2.3.0 diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh index 51588160..1db58883 100644 --- a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh +++ b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh @@ -212,11 +212,14 @@ revive() { _revive } +# in v1.19.8 metrics are changed from: +# kubelet_running_container_count => kubelet_running_containers +# kubelet_running_pod_count => kubelet_running_pods measure() { isClusterInitialized && isNodeInitialized || return 0 local -r regex="$(sed 's/^\s*//g' <<< ' - kubelet_running_container_count{container_state="running"} - kubelet_running_pod_count + kubelet_running_containers{container_state="running"} + kubelet_running_pods ' | paste -sd'|' | sed 's/^|/^(/; s/|$/)/')" runKubectl get -s https://localhost:10250 --raw /metrics --insecure-skip-tls-verify | egrep "$regex" | sed -r 's/\{[^}]+\}//g; s/ /: /g' | yq -j r - } diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index 9f6b254c..e078d7e2 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -18,7 +18,7 @@ "upgrading_policy": "in-place-parallel", "in-place-upgrade-nodes": [{ "container":{ - "snapshot": "ss-8ss6qpy1", + "snapshot": "ss-j8mi12ws", "zone": "pek3" }, "copy":[{ @@ -44,7 +44,7 @@ }] }, { "container":{ - "snapshot": "ss-s85qohr2", + "snapshot": "ss-3d9hmkwk", "zone": "pek3" }, "copy":[{ @@ -58,7 +58,7 @@ "role": "master", "container": { "type": "kvm", - "image": "img-y3b5ut5q", + "image": "img-vcu8pncd", "zone": "pek3" }, "instance_class": {{cluster.master.instance_class}}, @@ -144,7 +144,7 @@ "role": "node_perf", "container": { "type": "kvm", - "image": "img-y3b5ut5q", + "image": "img-vcu8pncd", "zone": "pek3" }, "instance_class": {{cluster.node_perf.instance_class}}, @@ -206,7 +206,7 @@ "role": "node_super_perf", "container": { "type": "kvm", - "image": "img-y3b5ut5q", + "image": "img-vcu8pncd", "zone": "pek3" }, "instance_class": {{cluster.node_super_perf.instance_class}}, @@ -268,7 +268,7 @@ "role": "node_gpu", "container": { "type": "kvm", - "image": "img-9h8jlw6j", + "image": "img-kirp1glw", "zone": "pek3" }, "instance_class": {{cluster.node_gpu.instance_class}}, @@ -382,13 +382,13 @@ "enable": true, "cmd": "appctl measure", "items": { - "kubelet_running_container_count": { + "kubelet_running_containers": { "unit": "", "value_type": "int", "statistics_type": "latest", "scale_factor_when_display": 1 }, - "kubelet_running_pod_count": { + "kubelet_running_pods": { "unit": "", "value_type": "int", "statistics_type": "latest", @@ -396,10 +396,10 @@ } }, "groups": { - "kubelet": ["kubelet_running_pod_count", "kubelet_running_container_count"] + "kubelet": ["kubelet_running_pods", "kubelet_running_containers"] }, "display": ["kubelet"], - "alarm": ["kubelet_running_container_count"] + "alarm": ["kubelet_running_containers"] }, "endpoints": { "nodeport": { diff --git a/app/locale/zh-cn.json b/app/locale/zh-cn.json index f74b5a51..7ba15e54 100644 --- a/app/locale/zh-cn.json +++ b/app/locale/zh-cn.json @@ -85,6 +85,8 @@ "parameters for kubelet": "kubelet 参数，自定义配置，支持多项配置，需严格遵循每行配置一项且保持 `key=value` 的格式，配置示例：`--add-dir-header=true`，默认值 `--eviction-hard=memory.available<5%` 表示当节点剩余内存不足 5% 时 kubelet 会立即关掉选中的容器组来释放内存，`--eviction-soft=memory.available<10%` 与 `--eviction-soft-grace-period=memory.available=2m` 表示当可用内存连续 2 分钟不足 10% 时，会平滑关闭（graceful shutdown）选中的容器组；注意 `--eviction-soft` 与 `--eviction-soft-grace-period` 必须同时指定，否则 kubelet 将无法启动；其他配置项请参考官方文档 [kubelet configurations](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/)，使用时请保留 `--` 符号", "kubelet_running_container_count": "正在运行的容器数量", "kubelet_running_pod_count": "正在运行的 Pod 数量", + "kubelet_running_containers": "正在运行的容器数量", + "kubelet_running_pods": "正在运行的 Pod 数量", "Fluent forward server": "Fluent 日志转发服务", "The fluent log server address to forward server, format host:port": "Fluent 日志服务器，用于将 Kubernetes 收集到的日志转发到用户自定义的日志服务，格式 host:port", "The Docker hub registry mirrors, use a blank to split multi registry mirrors": "完整的 Docker 镜像服务地址，比如 https://mirror.harbor.local；多个地址之间用空格隔开", From b0787058c3b0ef831a8681f6fa0300da9a432ed1 Mon Sep 17 00:00:00 2001 From: Jeff Date: Tue, 18 May 2021 19:58:50 +0800 Subject: [PATCH 05/13] fix client image Signed-off-by: Jeff --- app/cluster.json.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index e078d7e2..384a12cc 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -329,7 +329,7 @@ "role": "client", "container": { "type": "kvm", - "image": "img-5jbi2wg2", + "image": "img-swxveyde", "zone": "pek3" }, "instance_class": {{cluster.client.instance_class}}, From 1601cdbf9bc41aca37af4c43450b27bf64ca4456 Mon Sep 17 00:00:00 2001 From: Jeff Date: Wed, 19 May 2021 10:08:55 +0800 Subject: [PATCH 06/13] add multi zone policy Signed-off-by: Jeff --- app/cluster.json.mustache | 7 ++++--- app/config.json | 21 +++++++++++---------- app/locale/zh-cn.json | 1 + 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index 384a12cc..6afada54 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -6,6 +6,7 @@ "etcd_service": {{cluster.etcd_service}}, "elk_service": {{cluster.elk_service}} }, + "multi_zone_policy": "round_robin", "need_tag": true, "need_intranet_api_server": true, "exclude_node_columns": ["instance_class"], @@ -58,7 +59,7 @@ "role": "master", "container": { "type": "kvm", - "image": "img-vcu8pncd", + "image": "img-1sx7y8fs", "zone": "pek3" }, "instance_class": {{cluster.master.instance_class}}, @@ -144,7 +145,7 @@ "role": "node_perf", "container": { "type": "kvm", - "image": "img-vcu8pncd", + "image": "img-1sx7y8fs", "zone": "pek3" }, "instance_class": {{cluster.node_perf.instance_class}}, @@ -206,7 +207,7 @@ "role": "node_super_perf", "container": { "type": "kvm", - "image": "img-vcu8pncd", + "image": "img-1sx7y8fs", "zone": "pek3" }, "instance_class": {{cluster.node_super_perf.instance_class}}, diff --git a/app/config.json b/app/config.json index fda08900..ff43fcce 100644 --- a/app/config.json +++ b/app/config.json @@ -28,6 +28,7 @@ "range": [ "standard dev", "high-performance test", + "standard prod", "high-performance prod" ] }, { @@ -78,7 +79,7 @@ "range": [101, 202], "default": 202, "required": "yes", - "resource_group": [101, 202, 202] + "resource_group": [101, 202, 101, 202] }, { "key": "count", "label": "count", @@ -88,7 +89,7 @@ "range": [1, 3], "required": "yes", "changeable": false, - "resource_group": [1, 1, 3] + "resource_group": [1, 1, 3, 3] }, { "key": "cpu_model", "label": "CPU Model", @@ -105,7 +106,7 @@ "default": 8, "range": [4, 8, 12, 16, 24, 32, 64], "required": "yes", - "resource_group": [4, 4, 8] + "resource_group": [4, 4, 8, 8] }, { "key": "memory", "label": "Memory", @@ -114,7 +115,7 @@ "default": 16384, "range": [8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, 131072, 196608, 262144], "required": "yes", - "resource_group": [8192, 8192, 16384] + "resource_group": [8192, 8192, 16384, 16384] }, { "key": "volume_size", "label": "volume size", @@ -150,7 +151,7 @@ "min": 0, "max": 100, "required": "yes", - "resource_group": [2, 0, 0] + "resource_group": [2, 0, 3, 0] }, { "key": "cpu_model", "label": "CPU Model", @@ -210,7 +211,7 @@ "min": 0, "max": 100, "required": "yes", - "resource_group": [0, 2, 3] + "resource_group": [0, 2, 0, 3] }, { "key": "cpu_model", "label": "CPU Model", @@ -268,7 +269,7 @@ "min": 0, "max": 100, "default": 0, - "resource_group": [0, 0, 0], + "resource_group": [0, 0, 0, 0], "auto_scale_step": 1, "required": "yes" }, { @@ -337,7 +338,7 @@ "range": [101, 202], "default": 202, "required": "yes", - "resource_group": [101, 202, 202] + "resource_group": [101, 202, 101, 202] }, { "key": "cpu", "label": "CPU", @@ -346,7 +347,7 @@ "default": 2, "range": [1, 2], "required": "yes", - "resource_group": [1, 2, 2] + "resource_group": [1, 2, 1, 2] }, { "key": "memory", "label": "Memory", @@ -355,7 +356,7 @@ "default": 4096, "range": [1024, 2048, 4096], "required": "yes", - "resource_group": [2048, 4096, 4096] + "resource_group": [2048, 4096, 2048, 4096] }, { "key": "count", "label": "count", diff --git a/app/locale/zh-cn.json b/app/locale/zh-cn.json index 7ba15e54..6d7d47d7 100644 --- a/app/locale/zh-cn.json +++ b/app/locale/zh-cn.json @@ -129,6 +129,7 @@ "Resource Configuration": "快速配置", "standard dev": "基础型开发环境", "high-performance test": "企业型测试环境", + "standard prod": "基础性生产环境", "high-performance prod": "企业型生产环境", "The resource configuration of the service. Single master cluster cannot upgrade to HA mster cluster.": "请选择合适的预制资源配置类型，快速定义集群配置。也可根据自身需求自定义节点配置。非高可用集群不可以升级到高可用集群", "host aliases": "主机 hosts 记录", From a048891ffb30405ac4de106339c6ad15e7763e80 Mon Sep 17 00:00:00 2001 From: Jeff Date: Thu, 20 May 2021 17:29:09 +0800 Subject: [PATCH 07/13] fix health check failed when human readable node name disabled Signed-off-by: Jeff --- .../files/opt/app/current/bin/node/k8s-ctl.sh | 16 +++++++++++++++- app/config.json | 2 +- app/locale/en.json | 3 ++- app/locale/zh-cn.json | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh index 1db58883..79fb83ca 100644 --- a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh +++ b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh @@ -196,7 +196,7 @@ upgrade() { check() { _check - checkNodeStats '$2~/^Ready/' $MY_NODE_NAME + checkNodeStats '$2~/^Ready/' $(getMyNodeName) } checkSvc() { @@ -557,6 +557,11 @@ setUpNodeLocalDns() { sed "$replaceRules" /opt/app/current/conf/k8s/nodelocaldns-$K8S_VERSION.yml | runKubectl apply -f - } +countUnBoundPVCs() { + count=$(runKubectl get pvc -A --no-headers | grep -v Bound | wc -l) + return ${count} +} + _setUpStorage() { # remove previous version if $IS_UPGRADING_FROM_V2; then @@ -566,6 +571,15 @@ _setUpStorage() { # CSI plugin local -r csiChartFile=/opt/app/current/conf/k8s/csi-qingcloud-$QINGCLOUD_CSI_VERSION.tgz local -r csiValuesFile=/opt/app/current/conf/k8s/csi-qingcloud-values.yml + + # Need to uninstall and reinstall if upgrading, because helm upgrade will fail due to + # immutable fields change during upgrade. + if $IS_UPGRADING; then + # make sure there no pending pvs, if not skip upgrading csi-qingcloud + retry 600 1 0 countUnBoundPVCs || return 0 + runHelm -n kube-system uninstall csi-qingcloud + fi + yq p $QINGCLOUD_CONFIG config | cat - $csiValuesFile | \ runHelm -n kube-system upgrade --install csi-qingcloud $csiChartFile -f - diff --git a/app/config.json b/app/config.json index ff43fcce..4f6f73c5 100644 --- a/app/config.json +++ b/app/config.json @@ -21,7 +21,7 @@ }, { "key": "resource_group", "label": "Resource Configuration", - "description": "The resource configuration of the service. Single master cluster cannot upgrade to HA mster cluster.", + "description": "Resource Group Description", "disable_when_scale": true, "type": "string", "default": "high-performance test", diff --git a/app/locale/en.json b/app/locale/en.json index 07c036f5..b1f33329 100644 --- a/app/locale/en.json +++ b/app/locale/en.json @@ -3,5 +3,6 @@ "KS is not installed.": "KubeSphere is not installed.\nTo install KubeSphere, you can refer to [docs](https://docs.qingcloud.com/product/container/qke/), or [submit a ticket](https://console.qingcloud.com/tickets/) for support.", "Using master node IP. Please try again later when external IP is ready.": "The LB with EIP is being created, please refresh this tab later; currently the IP of the first control plane is displayed; if this message appears for a long time, please check the ks-console service under kubesphere-system namespace", "API access key id": "QingCloud IaaS [API Access Key](https://console.qingcloud.com/access_keys/), which will be used to create QingCloud resources, such as load balancers, volumes, etc.", - "Whether to install kubesphere": "Whether to install KubeSphere, KubeSphere and monitoring components will be installed by default;To install KubeSphere, you can refer to [docs](https://docs.qingcloud.com/product/container/qke/), or [submit a ticket](https://console.qingcloud.com/tickets/) for support." + "Whether to install kubesphere": "Whether to install KubeSphere, KubeSphere and monitoring components will be installed by default;To install KubeSphere, you can refer to [docs](https://docs.qingcloud.com/product/container/qke/), or [submit a ticket](https://console.qingcloud.com/tickets/) for support.", + "Resource Group Description": "The resource configuration of the service. Single master cluster cannot upgrade to HA mster cluster." } diff --git a/app/locale/zh-cn.json b/app/locale/zh-cn.json index 6d7d47d7..35bab189 100644 --- a/app/locale/zh-cn.json +++ b/app/locale/zh-cn.json @@ -131,7 +131,7 @@ "high-performance test": "企业型测试环境", "standard prod": "基础性生产环境", "high-performance prod": "企业型生产环境", - "The resource configuration of the service. Single master cluster cannot upgrade to HA mster cluster.": "请选择合适的预制资源配置类型，快速定义集群配置。也可根据自身需求自定义节点配置。非高可用集群不可以升级到高可用集群", + "Resource Group Description": "请选择合适的预制资源配置类型，快速定义集群配置。也可根据自身需求自定义节点配置。非高可用集群不可以升级到高可用集群", "host aliases": "主机 hosts 记录", "Set host aliases": "自定义添加到 /etc/hosts 文件的记录，比如 '192.168.2.2 host1,192.168.2.3 host2'，多条记录用逗号分割", "The insecure Docker registry, use a blank to split multi registry": "需要通过非安全的 HTTP 或不受信任的 HTTPS 访问的 Docker 仓库，比如 mirror.harbor.local，多个地址通过空格切分", From a7fea329aea67ee61b805f95994d0ba7e4c341d9 Mon Sep 17 00:00:00 2001 From: Jeff Date: Mon, 24 May 2021 10:02:52 +0800 Subject: [PATCH 08/13] skip webhook config file if provided configuration is empty Signed-off-by: Jeff --- .../files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl index 381f0f8e..d5ec4c23 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl @@ -34,7 +34,9 @@ apiServer: advertise-address: 0.0.0.0 {{- if $kubeAuditPolicy }} audit-policy-file: $KUBE_AUDIT_POLICY_RUNTIME_FILE + {{- if $kubeAuditWebhook }} audit-webhook-config-file: $KUBE_AUDIT_WEBHOOK_RUNTIME_FILE + {{- end }} audit-log-maxage: "{{ getv "/env/keep_audit_days" }}" audit-log-path: /etc/kubernetes/audit/logs/audit.log audit-log-maxsize: "{{ getv "/env/kube_audit_log_maxsize" "1" }}" From 5715e4b11c27a1e5f7504c834b5df3ade0604510 Mon Sep 17 00:00:00 2001 From: Jeff Date: Mon, 24 May 2021 10:54:03 +0800 Subject: [PATCH 09/13] add topology labels to nodes Signed-off-by: Jeff --- .../files/etc/confd/templates/kube.sh/01.node.env.tmpl | 3 +++ .../app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl index 6a262b81..33e9f8d1 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl @@ -44,6 +44,7 @@ ALL_NODES_EOF myRole={{ getv "/host/role" }} mySid={{ getv "/host/sid" }} myInstanceId={{ getv "/host/instance_id" }} +myZone={{ getv "/host/zone" }} myNodeName=$(buildNodeName $myRole $mySid) flush /opt/app/current/bin/envs/node.env << NODE_ENV_EOF @@ -53,10 +54,12 @@ CLUSTER_TAG={{ join (getvs "/cluster/cluster_tag") "" }} CLUSTER_VXNET={{ getv "/cluster/vxnet" }} CLUSTER_ZONE={{ getv "/cluster/zone" }} CLUSTER_API_SERVER={{ getv "/cluster/api_server/host" "api.qingcloud.com" }} +CLUSTER_REGION={{ getv "/cluster/region_id" }} MY_IP={{ getv "/host/ip" }} MY_SID=$mySid MY_ROLE=$myRole MY_NODE_NAME=$myNodeName +MY_ZONE=$myZone MY_INSTANCE_ID=$myInstanceId UPGRADED_FROM_V1={{ $upgradedFromV1 }} LB_IP_FROM_V1={{ $lbIpFromV1 }} diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh index 79fb83ca..27b92704 100644 --- a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh +++ b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh @@ -73,6 +73,7 @@ initCluster() { warmUpLocalDns if isFirstMaster; then initFirstNode; else initOtherNode; fi annotateInstanceId + labelTopology rm -rf $JOIN_CMD_FILE log --debug "done initializing cluster!" } @@ -907,6 +908,11 @@ annotateInstanceId() { runKubectl annotate no $(getMyNodeName) node.beta.kubernetes.io/instance-id="$MY_INSTANCE_ID" } +labelTopology() { + runKubectl label no $(getMyNodeName) topology.kubernetes.io/zone="$MY_ZONE" + runKubectl label no $(getMyNodeName) topology.kubernetes.io/region="$CLUSTER_REGION" +} + markAllInOne() { local hostName; hostName=${1:-$(getMyNodeName)} runKubectl taint node $hostName node-role.kubernetes.io/master:NoSchedule- From 373a13403dbe8709ac39d6b9b1faff0930f8cf33 Mon Sep 17 00:00:00 2001 From: Jeff Date: Tue, 25 May 2021 17:21:04 +0800 Subject: [PATCH 10/13] remove multi zone policy and update images Signed-off-by: Jeff --- app/cluster.json.mustache | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index 6afada54..799dcd68 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -6,20 +6,18 @@ "etcd_service": {{cluster.etcd_service}}, "elk_service": {{cluster.elk_service}} }, - "multi_zone_policy": "round_robin", "need_tag": true, "need_intranet_api_server": true, "exclude_node_columns": ["instance_class"], "advanced_actions": ["scale_horizontal"], "unsupported_actions": ["rollback"], "upgrade_policy": [ - "appv-egh21tjg", - "appv-39iiioy3" + "appv-egh21tjg" ], "upgrading_policy": "in-place-parallel", "in-place-upgrade-nodes": [{ "container":{ - "snapshot": "ss-j8mi12ws", + "snapshot": "ss-pned1k8r", "zone": "pek3" }, "copy":[{ @@ -45,7 +43,7 @@ }] }, { "container":{ - "snapshot": "ss-3d9hmkwk", + "snapshot": "ss-mixs6spp", "zone": "pek3" }, "copy":[{ @@ -59,7 +57,7 @@ "role": "master", "container": { "type": "kvm", - "image": "img-1sx7y8fs", + "image": "img-sve8d2fj", "zone": "pek3" }, "instance_class": {{cluster.master.instance_class}}, @@ -145,7 +143,7 @@ "role": "node_perf", "container": { "type": "kvm", - "image": "img-1sx7y8fs", + "image": "img-sve8d2fj", "zone": "pek3" }, "instance_class": {{cluster.node_perf.instance_class}}, @@ -207,7 +205,7 @@ "role": "node_super_perf", "container": { "type": "kvm", - "image": "img-1sx7y8fs", + "image": "img-sve8d2fj", "zone": "pek3" }, "instance_class": {{cluster.node_super_perf.instance_class}}, From dc31323907e4a07d9d6b2ea3646cb39e79e237a8 Mon Sep 17 00:00:00 2001 From: Jeff Date: Thu, 27 May 2021 16:37:02 +0800 Subject: [PATCH 11/13] fix cluster region_id missing when deploy in single zone Signed-off-by: Jeff --- .../files/etc/confd/templates/kube.sh/01.node.env.tmpl | 2 +- .../app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl index 33e9f8d1..a8419aea 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl @@ -54,7 +54,7 @@ CLUSTER_TAG={{ join (getvs "/cluster/cluster_tag") "" }} CLUSTER_VXNET={{ getv "/cluster/vxnet" }} CLUSTER_ZONE={{ getv "/cluster/zone" }} CLUSTER_API_SERVER={{ getv "/cluster/api_server/host" "api.qingcloud.com" }} -CLUSTER_REGION={{ getv "/cluster/region_id" }} +{{- if exists "/cluster/region_id" }}CLUSTER_REGION={{ getv "/cluster/region_id" }}{{- end }} MY_IP={{ getv "/host/ip" }} MY_SID=$mySid MY_ROLE=$myRole diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh index 27b92704..2392077c 100644 --- a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh +++ b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh @@ -909,8 +909,10 @@ annotateInstanceId() { } labelTopology() { - runKubectl label no $(getMyNodeName) topology.kubernetes.io/zone="$MY_ZONE" - runKubectl label no $(getMyNodeName) topology.kubernetes.io/region="$CLUSTER_REGION" + runKubectl label no $(getMyNodeName) topology.kubernetes.io/zone="$MY_ZONE" --overwrite + if [ ! -z "${CLUSTER_REGION}" ]; then + runKubectl label no $(getMyNodeName) topology.kubernetes.io/region="$CLUSTER_REGION" --overwrite + fi } markAllInOne() { From b9c553c8cdebfd81641905cfe99a53b6572e29f2 Mon Sep 17 00:00:00 2001 From: Jeff Date: Tue, 1 Jun 2021 10:28:12 +0800 Subject: [PATCH 12/13] remove unused images Signed-off-by: Jeff --- ansible/group_vars/all.yml | 76 ++++++++++++++------------------------ 1 file changed, 27 insertions(+), 49 deletions(-) diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 94008fcb..99060def 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -33,10 +33,6 @@ binaries: docker_images_k8s: - kubesphere/cloud-controller-manager:v1.4.4 -- calico/cni:v3.15.1 -- calico/kube-controllers:v3.15.1 -- calico/node:v3.15.1 -- calico/pod2daemon-flexvol:v3.15.1 - coredns/coredns:1.6.9 - csiplugin/csi-attacher:v2.1.1 - csiplugin/csi-node-driver-registrar:v1.2.0 @@ -45,23 +41,11 @@ docker_images_k8s: - csiplugin/csi-resizer:v0.4.0 - csiplugin/csi-snapshotter:v2.0.1 - csiplugin/snapshot-controller:v2.0.1 -- kubesphere/hostnic:v1.0.0-alpha.5 - nvidia/k8s-device-plugin:1.0.0-beta4 - kubesphere/flannel:v0.12.0 -- kubesphere/kube-apiserver:v1.18.12 -- kubesphere/kube-controller-manager:v1.18.12 -- kubesphere/kube-proxy:v1.18.12 -- kubesphere/kube-scheduler:v1.18.12 - kubesphere/pause:3.2 docker_images_ks: -- istio/citadel:1.4.8 -- istio/galley:1.4.8 -- istio/kubectl:1.4.8 -- istio/mixer:1.4.8 -- istio/pilot:1.4.8 -- istio/proxyv2:1.4.8 -- istio/sidecar_injector:1.4.8 - jaegertracing/jaeger-agent:1.17 - jaegertracing/jaeger-collector:1.17 - jaegertracing/jaeger-es-index-cleaner:1.17.1 @@ -69,45 +53,17 @@ docker_images_ks: - jaegertracing/jaeger-query:1.17 - jenkins/jnlp-slave:3.27-1 - jimmidyson/configmap-reload:v0.3.0 -- kubesphere/alert-adapter:v3.0.0 -- kubesphere/alerting-dbinit:v3.0.0 -- kubesphere/alerting:v2.1.2 - kubesphere/fluentbit-operator:migrator -- kubesphere/fluentbit-operator:v0.2.0 -- kubesphere/fluent-bit:v1.4.6 -- kubesphere/jenkins-uc:v3.0.0 -- kubesphere/ks-apiserver:v3.0.0 -- kubesphere/ks-console:v3.0.0 -- kubesphere/ks-controller-manager:v3.0.0 -- kubesphere/ks-devops:flyway-v3.0.0 -- kubesphere/ks-installer:v3.1.0 -- kubesphere/ks-upgrade:v3.0.0 - kubesphere/kube-auditing-operator:v0.1.2 - kubesphere/kube-auditing-webhook:v0.1.2 - kubesphere/kube-events-exporter:v0.1.0 - kubesphere/kube-events-operator:v0.1.0 -- kubesphere/kube-events-ruler:v0.1.0 -- kubesphere/kubefed:v0.3.0 -- kubesphere/kube-state-metrics:v1.9.6 - kubesphere/log-sidecar-injector:1.1 - kubesphere/elasticsearch-curator:v5.7.6 -- kubesphere/metrics-server:v0.3.7 - kubesphere/nginx-ingress-controller:0.24.1 - kubesphere/node-exporter:ks-v0.18.1 -- kubesphere/notification:flyway_v2.1.2 -- kubesphere/notification-manager-operator:v0.1.0 -- kubesphere/notification-manager:v0.1.0 -- kubesphere/notification:v2.1.2 -- kubesphere/prometheus-config-reloader:v0.38.3 -- kubesphere/prometheus-operator:v0.38.3 -- kubesphere/tower:v0.1.0 - mirrorgooglecontainers/defaultbackend-amd64:1.4 -- openpitrix/generate-kubeconfig:v0.5.0 -- openpitrix/openpitrix:flyway-v0.5.0 -- openpitrix/openpitrix:v0.5.0 -- openpitrix/release-app:sha-303629d - prom/alertmanager:v0.21.0 -- prom/prometheus:v2.20.1 docker_images_k8s_new: - kubesphere/kube-apiserver:v1.19.8 @@ -126,11 +82,11 @@ docker_images_k8s_new: - csiplugin/csi-neonsan:v1.2.1 - csiplugin/csi-neonsan-ubuntu:v1.2.1 - csiplugin/csi-neonsan-centos:v1.2.1 -- kubesphere/cloud-controller-manager:v1.4.6 +- kubesphere/cloud-controller-manager:v1.4.7 +- kubesphere/hostnic:v1.0.0-alpha.5 +- kubesphere/metrics-server:v0.4.2 docker_images_ks_new: -- openebs/provisioner-localpv:2.3.0 -- openebs/linux-utils:2.3.0 - kubesphere/ks-apiserver:v3.1.0 - kubesphere/ks-console:v3.1.0 - kubesphere/ks-controller-manager:v3.1.0 @@ -141,14 +97,13 @@ docker_images_ks_new: - kubesphere/tower:v0.2.0 - kubesphere/prometheus-config-reloader:v0.42.1 - kubesphere/prometheus-operator:v0.42.1 +- kubesphere/kube-state-metrics:v1.9.7 - prom/prometheus:v2.26.0 - prom/node-exporter:v0.18.1 - kubesphere/ks-alerting-migration:v3.1.0 - kubesphere/notification-manager-operator:v1.0.0 - kubesphere/notification-manager:v1.0.0 -- kubesphere/metrics-server:v0.4.2 - kubesphere/kube-rbac-proxy:v0.8.0 -- kubesphere/kube-state-metrics:v1.9.7 - openebs/provisioner-localpv:2.3.0 - thanosio/thanos:v0.18.0 - grafana/grafana:7.4.3 @@ -182,3 +137,26 @@ docker_images_ks_new: - java:openjdk-8-jre-alpine - fluent/fluentd:v1.4.2-2.0 - perl:latest +- osixia/openldap:1.3.0 +- redis:5.0.5-alpine +- alpine:3.10.4 +- haproxy:2.0.4 +- nginx:1.14-alpine +- minio/minio:RELEASE.2019-08-07T01-59-21Z +- minio/mc:RELEASE.2019-08-07T23-14-43Z +- kubesphere/elasticsearch-oss:6.7.0-1 +- docker:19.03 +- kubesphere/s2irun:v2.1.1 +- kubesphere/builder-base:v3.1.0 +- kubesphere/builder-nodejs:v3.1.0 +- kubesphere/builder-maven:v3.1.0 +- kubesphere/builder-go:v3.1.0 +- kubesphere/s2i-binary:v2.1.0 +- kubesphere/tomcat85-java11-centos7:v2.1.0 +- kubesphere/tomcat85-java11-runtime:v2.1.0 +- kubesphere/tomcat85-java8-centos7:v2.1.0 +- kubesphere/tomcat85-java8-runtime:v2.1.0 +- kubesphere/java-11-centos7:v2.1.0 +- kubesphere/java-8-centos7:v2.1.0 +- kubesphere/java-8-runtime:v2.1.0 +- kubesphere/java-11-runtime:v2.1.0 From ea93d27c5772d72960eeb81857cc95494277ea64 Mon Sep 17 00:00:00 2001 From: Jeff Date: Fri, 4 Jun 2021 09:31:40 +0800 Subject: [PATCH 13/13] fix multicluster member jwtSecret lost after upgraded to 3.1 Signed-off-by: Jeff --- ansible/group_vars/all.yml | 9 ++++----- ansible/make.yml | 4 ++-- ansible/requirements.yml | 2 +- .../files/opt/app/current/bin/node/k8s-ctl.sh | 7 ++++++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 99060def..23fe1452 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -8,7 +8,7 @@ coredns_version: 1.6.9 calico_version: 3.16.3 hostnic_version: 1.0.0-alpha.5 qingcloud_csi_version: 1.2.7 -qingcloud_ccm_version: 1.4.6 +qingcloud_ccm_version: 1.4.7 ks_version: 3.1.0 ks_installer_image_tag: v3.1.0 helm_version: 3.2.1 @@ -32,7 +32,6 @@ binaries: - /opt/k8s docker_images_k8s: -- kubesphere/cloud-controller-manager:v1.4.4 - coredns/coredns:1.6.9 - csiplugin/csi-attacher:v2.1.1 - csiplugin/csi-node-driver-registrar:v1.2.0 @@ -43,7 +42,7 @@ docker_images_k8s: - csiplugin/snapshot-controller:v2.0.1 - nvidia/k8s-device-plugin:1.0.0-beta4 - kubesphere/flannel:v0.12.0 -- kubesphere/pause:3.2 +- kubesphere/pause:3.1 docker_images_ks: - jaegertracing/jaeger-agent:1.17 @@ -64,13 +63,14 @@ docker_images_ks: - kubesphere/node-exporter:ks-v0.18.1 - mirrorgooglecontainers/defaultbackend-amd64:1.4 - prom/alertmanager:v0.21.0 +- docker:19.03 docker_images_k8s_new: - kubesphere/kube-apiserver:v1.19.8 - kubesphere/kube-scheduler:v1.19.8 - kubesphere/kube-proxy:v1.19.8 - kubesphere/kube-controller-manager:v1.19.8 -- kubesphere/pause:3.1 +- kubesphere/pause:3.2 - kubesphere/etcd:v3.4.13 - calico/cni:v3.16.3 - calico/kube-controllers:v3.16.3 @@ -145,7 +145,6 @@ docker_images_ks_new: - minio/minio:RELEASE.2019-08-07T01-59-21Z - minio/mc:RELEASE.2019-08-07T23-14-43Z - kubesphere/elasticsearch-oss:6.7.0-1 -- docker:19.03 - kubesphere/s2irun:v2.1.1 - kubesphere/builder-base:v3.1.0 - kubesphere/builder-nodejs:v3.1.0 diff --git a/ansible/make.yml b/ansible/make.yml index 3fd336fb..c1dcb59b 100644 --- a/ansible/make.yml +++ b/ansible/make.yml @@ -4,7 +4,7 @@ vars: target_env: "{{ lookup('env', 'target') }}" roles: - - docker-1.0.8 + - docker-1.0.10 - docker-images - hosts: k8s-client,k8s-node,gpu-node @@ -34,7 +34,7 @@ target_env: "{{ lookup('env', 'target') }}" roles: - qingcloud-cli-1.0.7 - - docker-1.0.8 + - docker-1.0.10 - etcd-1.1.0 - k8s-node # put extra modules above diff --git a/ansible/requirements.yml b/ansible/requirements.yml index f173f900..0553952c 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -3,7 +3,7 @@ - src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.5.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.1.0.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/create-service-user-1.0.0.tar.gz -- src: https://qingcloudappcenter.github.io/ansible-roles/docker-1.0.8.tar.gz +- src: https://qingcloudappcenter.github.io/ansible-roles/docker-1.0.10.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/golang-1.0.3.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/etcd-1.1.0.tar.gz - src: https://qingcloudappcenter.github.io/ansible-roles/install-1.0.6.tar.gz diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh index 2392077c..921c3adc 100644 --- a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh +++ b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh @@ -645,7 +645,12 @@ launchKs() { buildKsDynamicConf() { local -r ksCfgDynamicFile=/opt/app/current/conf/k8s/ks-config.dynamic.yml - yq p $ksCfgDynamicFile spec + if $IS_UPGRADING; then + # components could be manually enabled + runKubectl -n kubesphere-system get cc ks-installer -o yaml | yq r - 'spec' | yq p - spec + else + yq p $ksCfgDynamicFile spec + fi } buildKsConf() {