Merge pull request #410 from zhangziren1988/3.1.1

upgrade qke to 3.1.1
QingCloudAppcenter · Sep 23, 2021 · 7e2c7e2 · 7e2c7e2
2 parents 861958a + 2a6fd90
commit 7e2c7e2
Show file tree

Hide file tree

Showing 9 changed files with 176 additions and 68 deletions.
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
@@ -2,15 +2,15 @@ app_version: 3.1.0
 gcr_mirror: kubesphere
 etcd_version: 3.4.13
 cni_version: 0.8.6
-k8s_version: 1.19.8
+k8s_version: 1.20.6
 flannel_version: 0.12.0
 coredns_version: 1.6.9
 calico_version: 3.16.3
 hostnic_version: 1.0.0-alpha.5
 qingcloud_csi_version: 1.2.7
 qingcloud_ccm_version: 1.4.7
-ks_version: 3.1.0
-ks_installer_image_tag: v3.1.0
+ks_version: 3.1.1
+ks_installer_image_tag: v3.1.1
 helm_version: 3.2.1
 helm_2to3_version: 0.8.0
 helm_stable_repo: https://charts.kubesphere.io/mirror
@@ -43,29 +43,6 @@ docker_images_k8s:
 - nvidia/k8s-device-plugin:1.0.0-beta4
 - kubesphere/flannel:v0.12.0
 - kubesphere/pause:3.1
-
-docker_images_ks:
-- jaegertracing/jaeger-agent:1.17
-- jaegertracing/jaeger-collector:1.17
-- jaegertracing/jaeger-es-index-cleaner:1.17.1
-- jaegertracing/jaeger-operator:1.17.1
-- jaegertracing/jaeger-query:1.17
-- jenkins/jnlp-slave:3.27-1
-- jimmidyson/configmap-reload:v0.3.0
-- kubesphere/fluentbit-operator:migrator
-- kubesphere/kube-auditing-operator:v0.1.2
-- kubesphere/kube-auditing-webhook:v0.1.2
-- kubesphere/kube-events-exporter:v0.1.0
-- kubesphere/kube-events-operator:v0.1.0
-- kubesphere/log-sidecar-injector:1.1
-- kubesphere/elasticsearch-curator:v5.7.6
-- kubesphere/nginx-ingress-controller:0.24.1
-- kubesphere/node-exporter:ks-v0.18.1
-- mirrorgooglecontainers/defaultbackend-amd64:1.4
-- prom/alertmanager:v0.21.0
-- docker:19.03
-
-docker_images_k8s_new:
 - kubesphere/kube-apiserver:v1.19.8
 - kubesphere/kube-scheduler:v1.19.8
 - kubesphere/kube-proxy:v1.19.8
@@ -86,7 +63,7 @@ docker_images_k8s_new:
 - kubesphere/hostnic:v1.0.0-alpha.5
 - kubesphere/metrics-server:v0.4.2
 
-docker_images_ks_new:
+docker_images_ks:
 - kubesphere/ks-apiserver:v3.1.0
 - kubesphere/ks-console:v3.1.0
 - kubesphere/ks-controller-manager:v3.1.0
@@ -159,3 +136,134 @@ docker_images_ks_new:
 - kubesphere/java-8-centos7:v2.1.0
 - kubesphere/java-8-runtime:v2.1.0
 - kubesphere/java-11-runtime:v2.1.0
+
+docker_images_k8s_new:
+- kubesphere/kube-apiserver:v1.20.6
+- kubesphere/kube-scheduler:v1.20.6
+- kubesphere/kube-proxy:v1.20.6
+- kubesphere/kube-controller-manager:v1.20.6
+- kubesphere/pause:3.1
+- kubesphere/pause:3.2
+- kubesphere/etcd:v3.4.13
+- calico/cni:v3.16.3
+- calico/kube-controllers:v3.16.3
+- calico/node:v3.16.3
+- calico/pod2daemon-flexvol:v3.16.3
+- calico/typha:v3.16.3
+- kubesphere/flannel:v0.12.0
+- coredns/coredns:1.6.9
+- kubesphere/k8s-dns-node-cache:1.15.12
+- openebs/provisioner-localpv:2.10.1
+- openebs/linux-utils:2.10.0
+- kubesphere/nfs-client-provisioner:v3.1.0-k8s1.11
+
+docker_images_ks_new:
+- csiplugin/csi-neonsan:v1.2.0
+- csiplugin/csi-neonsan-ubuntu:v1.2.0
+- csiplugin/csi-neonsan-centos:v1.2.0
+- csiplugin/csi-provisioner:v1.5.0
+- csiplugin/csi-attacher:v2.1.1
+- csiplugin/csi-resizer:v0.4.0
+- csiplugin/csi-snapshotter:v2.0.1
+- csiplugin/csi-node-driver-registrar:v1.2.0
+- csiplugin/csi-qingcloud:v1.2.1
+- kubesphere/ks-apiserver:v3.1.1
+- kubesphere/ks-console:v3.1.1
+- kubesphere/ks-controller-manager:v3.1.1
+- kubespheredev/ks-installer:v3.1.1
+- kubesphere/kubectl:v1.20.0
+- kubesphere/kubectl:v1.19.1
+- redis:5.0.12-alpine
+- alpine:3.14
+- haproxy:2.0.22-alpine
+- nginx:1.14-alpine
+- minio/minio:RELEASE.2019-08-07T01-59-21Z
+- minio/mc:RELEASE.2019-08-07T23-14-43Z
+- mirrorgooglecontainers/defaultbackend-amd64:1.4
+- kubesphere/nginx-ingress-controller:v0.35.0
+- osixia/openldap:1.3.0
+- csiplugin/snapshot-controller:v3.0.3
+- kubesphere/kubefed:v0.7.0
+- kubesphere/tower:v0.2.0
+- kubesphere/prometheus-config-reloader:v0.42.1
+- kubesphere/prometheus-operator:v0.42.1
+- prom/alertmanager:v0.21.0
+- prom/prometheus:v2.26.0
+- prom/node-exporter:v0.18.1
+- kubesphere/ks-alerting-migration:v3.1.0
+- jimmidyson/configmap-reload:v0.3.0
+- kubesphere/notification-manager-operator:v1.0.0
+- kubesphere/notification-manager:v1.0.0
+- kubesphere/metrics-server:v0.4.2
+- kubesphere/kube-rbac-proxy:v0.8.0
+- kubesphere/kube-state-metrics:v1.9.7
+- openebs/provisioner-localpv:2.3.0
+- thanosio/thanos:v0.18.0
+- grafana/grafana:7.4.3
+- kubesphere/elasticsearch-oss:6.7.0-1
+- kubesphere/elasticsearch-curator:v5.7.6
+- kubesphere/fluentbit-operator:v0.5.0
+- kubesphere/fluentbit-operator:migrator
+- kubesphere/fluent-bit:v1.6.9
+- elastic/filebeat:6.7.0
+- kubesphere/kube-auditing-operator:v0.1.2
+- kubesphere/kube-auditing-webhook:v0.1.2
+- kubesphere/kube-events-exporter:v0.1.0
+- kubesphere/kube-events-operator:v0.1.0
+- kubesphere/kube-events-ruler:v0.2.0
+- kubesphere/log-sidecar-injector:1.1
+- docker:19.03
+- istio/pilot:1.6.10
+- istio/proxyv2:1.6.10
+- jaegertracing/jaeger-agent:1.17
+- jaegertracing/jaeger-collector:1.17
+- jaegertracing/jaeger-es-index-cleaner:1.17
+- jaegertracing/jaeger-operator:1.17.1
+- jaegertracing/jaeger-query:1.17
+- kubesphere/kiali:v1.26.1
+- kubesphere/kiali-operator:v1.26.1
+- kubesphere/ks-jenkins:2.249.1
+- jenkins/jnlp-slave:3.27-1
+- kubesphere/s2ioperator:v3.1.0
+- kubesphere/s2irun:v2.1.1
+- kubesphere/builder-base:v3.1.0
+- kubesphere/builder-nodejs:v3.1.0
+- kubesphere/builder-maven:v3.1.0
+- kubesphere/builder-go:v3.1.0
+- kubesphere/s2i-binary:v2.1.0
+- kubesphere/tomcat85-java11-centos7:v2.1.0
+- kubesphere/tomcat85-java11-runtime:v2.1.0
+- kubesphere/tomcat85-java8-centos7:v2.1.0
+- kubesphere/tomcat85-java8-runtime:v2.1.0
+- kubesphere/java-11-centos7:v2.1.0
+- kubesphere/java-8-centos7:v2.1.0
+- kubesphere/java-8-runtime:v2.1.0
+- kubesphere/java-11-runtime:v2.1.0
+- kubesphere/nodejs-8-centos7:v2.1.0
+- kubesphere/nodejs-6-centos7:v2.1.0
+- kubesphere/nodejs-4-centos7:v2.1.0
+- kubesphere/python-36-centos7:v2.1.0
+- kubesphere/python-35-centos7:v2.1.0
+- kubesphere/python-34-centos7:v2.1.0
+- kubesphere/python-27-centos7:v2.1.0
+- kubespheredev/openpitrix-jobs:v3.1.1
+- weaveworks/scope:1.13.0
+- kubeedge/cloudcore:v1.6.2
+- kubesphere/edge-watcher:v0.1.0
+- kubesphere/kube-rbac-proxy:v0.5.0
+- kubesphere/edge-watcher-agent:v0.1.0
+- kubesphere/examples-bookinfo-productpage-v1:1.16.2
+- kubesphere/examples-bookinfo-reviews-v1:1.16.2
+- kubesphere/examples-bookinfo-reviews-v2:1.16.2
+- kubesphere/examples-bookinfo-reviews-v3:1.16.2
+- kubesphere/examples-bookinfo-details-v1:1.16.2
+- kubesphere/examples-bookinfo-ratings-v1:1.16.3
+- busybox:1.31.1
+- joosthofman/wget:1.0
+- kubesphere/netshoot:v1.0
+- nginxdemos/hello:plain-text
+- wordpress:4.8-apache
+- mirrorgooglecontainers/hpa-example:latest
+- java:openjdk-8-jre-alpine
+- fluent/fluentd:v1.4.2-2.0
+- perl:latest
diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/03.hosts.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/03.hosts.tmpl
@@ -15,10 +15,17 @@ printHostEntry() {
   echo $1$'\t'$2
 }
 
+{{- $myZone := getv "/host/zone" }}
+{{- $iaasApiServer := map "pek3" "10.140.24.6" "pek3a" "10.91.84.7" "pek3b" "10.140.24.6" "pek3c" "10.140.24.6" "pek3d" "10.140.24.6" "pekt3" "10.181.0.34" "pekt3d" "10.181.0.34" "sh1" "10.120.47.8" "sh1a" "10.120.47.8" "sh1b" "10.120.47.8" "gd2" "10.150.21.8" "gd2a" "10.150.21.8" "gd2b" "10.150.21.8" "ap2a" "10.160.3.4" "ap3a" "10.200.1.13"}}
+staticApiIp={{ index $iaasApiServer $myZone }}
+apiServer={{ getv "/cluster/api_server/host" "ks.api.qingcloud.com" }}
+dynamicApiIp=$(dig +timeout=2 +short $apiServer | grep -o "^[0-9.]\+")
+
 flush >> $hostsFile.swap << HOSTS_FILE
 # >> QKE nodes. WARNING: this is managed by script and please don't touch manually.
 $(printHostEntry 127.0.1.1 $myNodeName)
 $(printHostEntry 0.0.0.0 dl.k8s.io)
+$(printHostEntry ${dynamicApiIp:-$staticApiIp} $apiServer)
 $(printHostEntry ${lbIp:-$firstMasterIp} loadbalancer)
 $(echo "$allNodes" | awk -F/ '{printf("%s\t%s %s%s\n", $7, $4, $5, $2~/^n/ ? " "$2$3 : "")}')
 

diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/08.csi-sc.yml.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/08.csi-sc.yml.tmpl
@@ -1,7 +1,7 @@
 {{- $volume0 := map "min"  10 "max"  2000 "step"  10 }}
 {{- $volume2 := map "min" 100 "max"  5000 "step" 100 }}
-{{- $volume5 := map "min" 100 "max" 20000 "step" 100 }}
-{{- $volume6 := map "min" 100 "max" 10000 "step" 100 }}
+{{- $volume5 := map "min" 20 "max" 20000 "step" 10 }}
+{{- $volume6 := map "min" 20 "max" 10000 "step" 10 }}
 {{- $volumeClassMaps := map "0" $volume0 "2" $volume2 "3" $volume0 "5" $volume5 "6" $volume6 "100" $volume0 "200" $volume0 }}
 {{- $volumeNamesMap := map "0" "high-perf" "2" "high-capacity-legacy" "3" "super-high-perf" "5" "neonsan" "6" "high-capacity" "100" "standard" "200" "ssd-enterprise" }}
 

diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh
@@ -175,6 +175,8 @@ upgrade() {
         retry 10 1 0 fixDns
       fi
       applyKubeProxyLogLevel
+      # restart metrics-server to be ready to avoid https://github.com/kubernetes/kubernetes/pull/96371
+      runKubectl -n kube-system rollout restart deploy metrics-server
       setUpNetwork
       setUpCloudControllerMgr
       execute setUpStorage
@@ -579,6 +581,7 @@ _setUpStorage() {
     # make sure there no pending pvs, if not skip upgrading csi-qingcloud
     retry 600 1 0 countUnBoundPVCs || return 0
     runHelm -n kube-system uninstall csi-qingcloud
+    runKubectl delete -f /opt/app/current/conf/k8s/csi-sc.yml
   fi
 
   yq p $QINGCLOUD_CONFIG config | cat - $csiValuesFile | \

diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/conf/systemd/kube-lb.service b/ansible/roles/app-role-k8s/files/opt/app/current/conf/systemd/kube-lb.service
@@ -1,5 +1,7 @@
 [Unit]
 Description=create kube-apiserver load balancer
+Wants=network-online.target
+After=network-online.target
 
 [Service]
 ExecStart=/usr/bin/appctl setUpKubeLb

diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache
@@ -12,12 +12,14 @@
   "advanced_actions": ["scale_horizontal"],
   "unsupported_actions": ["rollback"],
   "upgrade_policy": [
-    "appv-egh21tjg"
+    "appv-doh2u06i",
+    "appv-egh21tjg",
+    "appv-0exsrgnh"
   ],
   "upgrading_policy": "in-place-parallel",
   "in-place-upgrade-nodes": [{
     "container":{
-      "snapshot": "ss-pned1k8r",
+      "snapshot": "ss-efvwtpln",
       "zone": "pek3"
     },
     "copy":[{
@@ -43,7 +45,7 @@
     }]
   }, {
     "container":{
-      "snapshot": "ss-mixs6spp",
+      "snapshot": "ss-58hyhs7i",
       "zone": "pek3"
     },
     "copy":[{
@@ -57,7 +59,7 @@
     "role": "master",
     "container": {
       "type": "kvm",
-      "image": "img-sve8d2fj",
+      "image": "img-e2vxagq6",
       "zone": "pek3"
     },
     "instance_class": {{cluster.master.instance_class}},
@@ -81,7 +83,8 @@
       "size": {{cluster.master.volume_size}},
       "mount_point": "/data",
       "mount_options": "defaults,noatime",
-      "filesystem": "ext4"
+      "filesystem": "ext4",
+      "class": {{cluster.master.volume_class}}
     },
     "services": {
       "start": {
@@ -114,7 +117,7 @@
       },
       "upgrade": {
         "order": 1,
-        "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' || (exit 151); fi",
+        "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' && test $(netstat -nltp | grep sshd  | awk '{print $4}'|awk -F : '{print $2}') -eq 22 || (exit 151); fi",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 10800
       },
@@ -143,7 +146,7 @@
     "role": "node_perf",
     "container": {
       "type": "kvm",
-      "image": "img-sve8d2fj",
+      "image": "img-e2vxagq6",
       "zone": "pek3"
     },
     "instance_class": {{cluster.node_perf.instance_class}},
@@ -185,7 +188,7 @@
       },
       "upgrade": {
         "order": 1,
-        "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' || (exit 151); fi",
+        "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' && test $(netstat -nltp | grep sshd  | awk '{print $4}'|awk -F : '{print $2}') -eq 22 || (exit 151); fi",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 10800
       },
@@ -205,7 +208,7 @@
     "role": "node_super_perf",
     "container": {
       "type": "kvm",
-      "image": "img-sve8d2fj",
+      "image": "img-e2vxagq6",
       "zone": "pek3"
     },
     "instance_class": {{cluster.node_super_perf.instance_class}},
@@ -247,7 +250,7 @@
       },
       "upgrade": {
         "order": 1,
-        "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' || (exit 151); fi",
+        "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' && test $(netstat -nltp | grep sshd  | awk '{print $4}'|awk -F : '{print $2}') -eq 22 || (exit 151); fi",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 10800
       },
@@ -267,7 +270,7 @@
     "role": "node_gpu",
     "container": {
       "type": "kvm",
-      "image": "img-kirp1glw",
+      "image": "img-q0kubgen",
       "zone": "pek3"
     },
     "instance_class": {{cluster.node_gpu.instance_class}},
@@ -311,7 +314,7 @@
       },
       "upgrade": {
         "order": 1,
-        "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' || (exit 151); fi",
+        "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' && test $(netstat -nltp | grep sshd  | awk '{print $4}'|awk -F : '{print $2}') -eq 22 || (exit 151); fi",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 10800
       },
@@ -328,7 +331,7 @@
     "role": "client",
     "container": {
       "type": "kvm",
-      "image": "img-swxveyde",
+      "image": "img-frbcv58s",
       "zone": "pek3"
     },
     "instance_class": {{cluster.client.instance_class}},
@@ -351,6 +354,7 @@
       },
       "upgrade": {
         "order": 2,
+        "pre_check": "if test $(netstat -nltp | grep sshd  | awk '{print $4}'|awk -F : '{print $2}') -ne 22 ; then  (exit 151);fi",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 43200
       },

diff --git a/app/config.json b/app/config.json
@@ -116,6 +116,14 @@
         "range": [8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, 131072, 196608, 262144],
         "required": "yes",
         "resource_group": [8192, 8192, 16384, 16384]
+      }, {
+        "key": "volume_class",
+        "label": "volume class",
+        "description": "The volume class",
+        "type": "integer",
+        "default": 6,
+        "range": [6],
+        "required": "yes"
       }, {
         "key": "volume_size",
         "label": "volume size",

diff --git a/app/locale/zh-cn.json b/app/locale/zh-cn.json
@@ -12,6 +12,8 @@
     "count": "数量",
     "Number of master for the cluster to create": "主节点数量",
     "resource type": "资源类型",
+    "volume class": "持久存储卷类型",
+    "The volume class": "持久存储卷类型",
     "volume size": "硬盘大小",
     "The volume size for each instance": "每个机器的硬盘大小",
     "load balancer": "负载均衡器",