From 2a6fd90fe06a65bd3407cc3a0fc30cc58410d402 Mon Sep 17 00:00:00 2001 From: zhangziren1988 Date: Thu, 23 Sep 2021 14:39:45 +0800 Subject: [PATCH] upgrade qke to 3.1.1 Signed-off-by: Nick --- ansible/group_vars/all.yml | 162 +++++++++++++++--- .../etc/confd/templates/kube.sh/03.hosts.tmpl | 7 + .../templates/kube.sh/08.csi-sc.yml.tmpl | 4 +- .../files/opt/app/current/bin/node/k8s-ctl.sh | 3 + .../app/current/conf/systemd/kube-lb.service | 2 + app/cluster.json.mustache | 30 ++-- app/config.json | 8 + app/locale/zh-cn.json | 2 + app/replace_policy.json | 26 --- 9 files changed, 176 insertions(+), 68 deletions(-) diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 23fe1452..f6b5af5c 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -2,15 +2,15 @@ app_version: 3.1.0 gcr_mirror: kubesphere etcd_version: 3.4.13 cni_version: 0.8.6 -k8s_version: 1.19.8 +k8s_version: 1.20.6 flannel_version: 0.12.0 coredns_version: 1.6.9 calico_version: 3.16.3 hostnic_version: 1.0.0-alpha.5 qingcloud_csi_version: 1.2.7 qingcloud_ccm_version: 1.4.7 -ks_version: 3.1.0 -ks_installer_image_tag: v3.1.0 +ks_version: 3.1.1 +ks_installer_image_tag: v3.1.1 helm_version: 3.2.1 helm_2to3_version: 0.8.0 helm_stable_repo: https://charts.kubesphere.io/mirror @@ -43,29 +43,6 @@ docker_images_k8s: - nvidia/k8s-device-plugin:1.0.0-beta4 - kubesphere/flannel:v0.12.0 - kubesphere/pause:3.1 - -docker_images_ks: -- jaegertracing/jaeger-agent:1.17 -- jaegertracing/jaeger-collector:1.17 -- jaegertracing/jaeger-es-index-cleaner:1.17.1 -- jaegertracing/jaeger-operator:1.17.1 -- jaegertracing/jaeger-query:1.17 -- jenkins/jnlp-slave:3.27-1 -- jimmidyson/configmap-reload:v0.3.0 -- kubesphere/fluentbit-operator:migrator -- kubesphere/kube-auditing-operator:v0.1.2 -- kubesphere/kube-auditing-webhook:v0.1.2 -- kubesphere/kube-events-exporter:v0.1.0 -- kubesphere/kube-events-operator:v0.1.0 -- kubesphere/log-sidecar-injector:1.1 -- kubesphere/elasticsearch-curator:v5.7.6 -- kubesphere/nginx-ingress-controller:0.24.1 -- kubesphere/node-exporter:ks-v0.18.1 -- mirrorgooglecontainers/defaultbackend-amd64:1.4 -- prom/alertmanager:v0.21.0 -- docker:19.03 - -docker_images_k8s_new: - kubesphere/kube-apiserver:v1.19.8 - kubesphere/kube-scheduler:v1.19.8 - kubesphere/kube-proxy:v1.19.8 @@ -86,7 +63,7 @@ docker_images_k8s_new: - kubesphere/hostnic:v1.0.0-alpha.5 - kubesphere/metrics-server:v0.4.2 -docker_images_ks_new: +docker_images_ks: - kubesphere/ks-apiserver:v3.1.0 - kubesphere/ks-console:v3.1.0 - kubesphere/ks-controller-manager:v3.1.0 @@ -159,3 +136,134 @@ docker_images_ks_new: - kubesphere/java-8-centos7:v2.1.0 - kubesphere/java-8-runtime:v2.1.0 - kubesphere/java-11-runtime:v2.1.0 + +docker_images_k8s_new: +- kubesphere/kube-apiserver:v1.20.6 +- kubesphere/kube-scheduler:v1.20.6 +- kubesphere/kube-proxy:v1.20.6 +- kubesphere/kube-controller-manager:v1.20.6 +- kubesphere/pause:3.1 +- kubesphere/pause:3.2 +- kubesphere/etcd:v3.4.13 +- calico/cni:v3.16.3 +- calico/kube-controllers:v3.16.3 +- calico/node:v3.16.3 +- calico/pod2daemon-flexvol:v3.16.3 +- calico/typha:v3.16.3 +- kubesphere/flannel:v0.12.0 +- coredns/coredns:1.6.9 +- kubesphere/k8s-dns-node-cache:1.15.12 +- openebs/provisioner-localpv:2.10.1 +- openebs/linux-utils:2.10.0 +- kubesphere/nfs-client-provisioner:v3.1.0-k8s1.11 + +docker_images_ks_new: +- csiplugin/csi-neonsan:v1.2.0 +- csiplugin/csi-neonsan-ubuntu:v1.2.0 +- csiplugin/csi-neonsan-centos:v1.2.0 +- csiplugin/csi-provisioner:v1.5.0 +- csiplugin/csi-attacher:v2.1.1 +- csiplugin/csi-resizer:v0.4.0 +- csiplugin/csi-snapshotter:v2.0.1 +- csiplugin/csi-node-driver-registrar:v1.2.0 +- csiplugin/csi-qingcloud:v1.2.1 +- kubesphere/ks-apiserver:v3.1.1 +- kubesphere/ks-console:v3.1.1 +- kubesphere/ks-controller-manager:v3.1.1 +- kubespheredev/ks-installer:v3.1.1 +- kubesphere/kubectl:v1.20.0 +- kubesphere/kubectl:v1.19.1 +- redis:5.0.12-alpine +- alpine:3.14 +- haproxy:2.0.22-alpine +- nginx:1.14-alpine +- minio/minio:RELEASE.2019-08-07T01-59-21Z +- minio/mc:RELEASE.2019-08-07T23-14-43Z +- mirrorgooglecontainers/defaultbackend-amd64:1.4 +- kubesphere/nginx-ingress-controller:v0.35.0 +- osixia/openldap:1.3.0 +- csiplugin/snapshot-controller:v3.0.3 +- kubesphere/kubefed:v0.7.0 +- kubesphere/tower:v0.2.0 +- kubesphere/prometheus-config-reloader:v0.42.1 +- kubesphere/prometheus-operator:v0.42.1 +- prom/alertmanager:v0.21.0 +- prom/prometheus:v2.26.0 +- prom/node-exporter:v0.18.1 +- kubesphere/ks-alerting-migration:v3.1.0 +- jimmidyson/configmap-reload:v0.3.0 +- kubesphere/notification-manager-operator:v1.0.0 +- kubesphere/notification-manager:v1.0.0 +- kubesphere/metrics-server:v0.4.2 +- kubesphere/kube-rbac-proxy:v0.8.0 +- kubesphere/kube-state-metrics:v1.9.7 +- openebs/provisioner-localpv:2.3.0 +- thanosio/thanos:v0.18.0 +- grafana/grafana:7.4.3 +- kubesphere/elasticsearch-oss:6.7.0-1 +- kubesphere/elasticsearch-curator:v5.7.6 +- kubesphere/fluentbit-operator:v0.5.0 +- kubesphere/fluentbit-operator:migrator +- kubesphere/fluent-bit:v1.6.9 +- elastic/filebeat:6.7.0 +- kubesphere/kube-auditing-operator:v0.1.2 +- kubesphere/kube-auditing-webhook:v0.1.2 +- kubesphere/kube-events-exporter:v0.1.0 +- kubesphere/kube-events-operator:v0.1.0 +- kubesphere/kube-events-ruler:v0.2.0 +- kubesphere/log-sidecar-injector:1.1 +- docker:19.03 +- istio/pilot:1.6.10 +- istio/proxyv2:1.6.10 +- jaegertracing/jaeger-agent:1.17 +- jaegertracing/jaeger-collector:1.17 +- jaegertracing/jaeger-es-index-cleaner:1.17 +- jaegertracing/jaeger-operator:1.17.1 +- jaegertracing/jaeger-query:1.17 +- kubesphere/kiali:v1.26.1 +- kubesphere/kiali-operator:v1.26.1 +- kubesphere/ks-jenkins:2.249.1 +- jenkins/jnlp-slave:3.27-1 +- kubesphere/s2ioperator:v3.1.0 +- kubesphere/s2irun:v2.1.1 +- kubesphere/builder-base:v3.1.0 +- kubesphere/builder-nodejs:v3.1.0 +- kubesphere/builder-maven:v3.1.0 +- kubesphere/builder-go:v3.1.0 +- kubesphere/s2i-binary:v2.1.0 +- kubesphere/tomcat85-java11-centos7:v2.1.0 +- kubesphere/tomcat85-java11-runtime:v2.1.0 +- kubesphere/tomcat85-java8-centos7:v2.1.0 +- kubesphere/tomcat85-java8-runtime:v2.1.0 +- kubesphere/java-11-centos7:v2.1.0 +- kubesphere/java-8-centos7:v2.1.0 +- kubesphere/java-8-runtime:v2.1.0 +- kubesphere/java-11-runtime:v2.1.0 +- kubesphere/nodejs-8-centos7:v2.1.0 +- kubesphere/nodejs-6-centos7:v2.1.0 +- kubesphere/nodejs-4-centos7:v2.1.0 +- kubesphere/python-36-centos7:v2.1.0 +- kubesphere/python-35-centos7:v2.1.0 +- kubesphere/python-34-centos7:v2.1.0 +- kubesphere/python-27-centos7:v2.1.0 +- kubespheredev/openpitrix-jobs:v3.1.1 +- weaveworks/scope:1.13.0 +- kubeedge/cloudcore:v1.6.2 +- kubesphere/edge-watcher:v0.1.0 +- kubesphere/kube-rbac-proxy:v0.5.0 +- kubesphere/edge-watcher-agent:v0.1.0 +- kubesphere/examples-bookinfo-productpage-v1:1.16.2 +- kubesphere/examples-bookinfo-reviews-v1:1.16.2 +- kubesphere/examples-bookinfo-reviews-v2:1.16.2 +- kubesphere/examples-bookinfo-reviews-v3:1.16.2 +- kubesphere/examples-bookinfo-details-v1:1.16.2 +- kubesphere/examples-bookinfo-ratings-v1:1.16.3 +- busybox:1.31.1 +- joosthofman/wget:1.0 +- kubesphere/netshoot:v1.0 +- nginxdemos/hello:plain-text +- wordpress:4.8-apache +- mirrorgooglecontainers/hpa-example:latest +- java:openjdk-8-jre-alpine +- fluent/fluentd:v1.4.2-2.0 +- perl:latest diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/03.hosts.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/03.hosts.tmpl index 70672b72..74231c12 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/03.hosts.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/03.hosts.tmpl @@ -15,10 +15,17 @@ printHostEntry() { echo $1$'\t'$2 } +{{- $myZone := getv "/host/zone" }} +{{- $iaasApiServer := map "pek3" "10.140.24.6" "pek3a" "10.91.84.7" "pek3b" "10.140.24.6" "pek3c" "10.140.24.6" "pek3d" "10.140.24.6" "pekt3" "10.181.0.34" "pekt3d" "10.181.0.34" "sh1" "10.120.47.8" "sh1a" "10.120.47.8" "sh1b" "10.120.47.8" "gd2" "10.150.21.8" "gd2a" "10.150.21.8" "gd2b" "10.150.21.8" "ap2a" "10.160.3.4" "ap3a" "10.200.1.13"}} +staticApiIp={{ index $iaasApiServer $myZone }} +apiServer={{ getv "/cluster/api_server/host" "ks.api.qingcloud.com" }} +dynamicApiIp=$(dig +timeout=2 +short $apiServer | grep -o "^[0-9.]\+") + flush >> $hostsFile.swap << HOSTS_FILE # >> QKE nodes. WARNING: this is managed by script and please don't touch manually. $(printHostEntry 127.0.1.1 $myNodeName) $(printHostEntry 0.0.0.0 dl.k8s.io) +$(printHostEntry ${dynamicApiIp:-$staticApiIp} $apiServer) $(printHostEntry ${lbIp:-$firstMasterIp} loadbalancer) $(echo "$allNodes" | awk -F/ '{printf("%s\t%s %s%s\n", $7, $4, $5, $2~/^n/ ? " "$2$3 : "")}') diff --git a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/08.csi-sc.yml.tmpl b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/08.csi-sc.yml.tmpl index 3ff061db..bceb2b52 100644 --- a/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/08.csi-sc.yml.tmpl +++ b/ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/08.csi-sc.yml.tmpl @@ -1,7 +1,7 @@ {{- $volume0 := map "min" 10 "max" 2000 "step" 10 }} {{- $volume2 := map "min" 100 "max" 5000 "step" 100 }} -{{- $volume5 := map "min" 100 "max" 20000 "step" 100 }} -{{- $volume6 := map "min" 100 "max" 10000 "step" 100 }} +{{- $volume5 := map "min" 20 "max" 20000 "step" 10 }} +{{- $volume6 := map "min" 20 "max" 10000 "step" 10 }} {{- $volumeClassMaps := map "0" $volume0 "2" $volume2 "3" $volume0 "5" $volume5 "6" $volume6 "100" $volume0 "200" $volume0 }} {{- $volumeNamesMap := map "0" "high-perf" "2" "high-capacity-legacy" "3" "super-high-perf" "5" "neonsan" "6" "high-capacity" "100" "standard" "200" "ssd-enterprise" }} diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh index 921c3adc..3b3438dc 100644 --- a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh +++ b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s-ctl.sh @@ -175,6 +175,8 @@ upgrade() { retry 10 1 0 fixDns fi applyKubeProxyLogLevel + # restart metrics-server to be ready to avoid https://github.com/kubernetes/kubernetes/pull/96371 + runKubectl -n kube-system rollout restart deploy metrics-server setUpNetwork setUpCloudControllerMgr execute setUpStorage @@ -579,6 +581,7 @@ _setUpStorage() { # make sure there no pending pvs, if not skip upgrading csi-qingcloud retry 600 1 0 countUnBoundPVCs || return 0 runHelm -n kube-system uninstall csi-qingcloud + runKubectl delete -f /opt/app/current/conf/k8s/csi-sc.yml fi yq p $QINGCLOUD_CONFIG config | cat - $csiValuesFile | \ diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/conf/systemd/kube-lb.service b/ansible/roles/app-role-k8s/files/opt/app/current/conf/systemd/kube-lb.service index 17756084..1d213f17 100644 --- a/ansible/roles/app-role-k8s/files/opt/app/current/conf/systemd/kube-lb.service +++ b/ansible/roles/app-role-k8s/files/opt/app/current/conf/systemd/kube-lb.service @@ -1,5 +1,7 @@ [Unit] Description=create kube-apiserver load balancer +Wants=network-online.target +After=network-online.target [Service] ExecStart=/usr/bin/appctl setUpKubeLb diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache index 799dcd68..c7808f97 100644 --- a/app/cluster.json.mustache +++ b/app/cluster.json.mustache @@ -12,12 +12,14 @@ "advanced_actions": ["scale_horizontal"], "unsupported_actions": ["rollback"], "upgrade_policy": [ - "appv-egh21tjg" + "appv-doh2u06i", + "appv-egh21tjg", + "appv-0exsrgnh" ], "upgrading_policy": "in-place-parallel", "in-place-upgrade-nodes": [{ "container":{ - "snapshot": "ss-pned1k8r", + "snapshot": "ss-efvwtpln", "zone": "pek3" }, "copy":[{ @@ -43,7 +45,7 @@ }] }, { "container":{ - "snapshot": "ss-mixs6spp", + "snapshot": "ss-58hyhs7i", "zone": "pek3" }, "copy":[{ @@ -57,7 +59,7 @@ "role": "master", "container": { "type": "kvm", - "image": "img-sve8d2fj", + "image": "img-e2vxagq6", "zone": "pek3" }, "instance_class": {{cluster.master.instance_class}}, @@ -81,7 +83,8 @@ "size": {{cluster.master.volume_size}}, "mount_point": "/data", "mount_options": "defaults,noatime", - "filesystem": "ext4" + "filesystem": "ext4", + "class": {{cluster.master.volume_class}} }, "services": { "start": { @@ -114,7 +117,7 @@ }, "upgrade": { "order": 1, - "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' || (exit 151); fi", + "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' && test $(netstat -nltp | grep sshd | awk '{print $4}'|awk -F : '{print $2}') -eq 22 || (exit 151); fi", "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh", "timeout": 10800 }, @@ -143,7 +146,7 @@ "role": "node_perf", "container": { "type": "kvm", - "image": "img-sve8d2fj", + "image": "img-e2vxagq6", "zone": "pek3" }, "instance_class": {{cluster.node_perf.instance_class}}, @@ -185,7 +188,7 @@ }, "upgrade": { "order": 1, - "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' || (exit 151); fi", + "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' && test $(netstat -nltp | grep sshd | awk '{print $4}'|awk -F : '{print $2}') -eq 22 || (exit 151); fi", "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh", "timeout": 10800 }, @@ -205,7 +208,7 @@ "role": "node_super_perf", "container": { "type": "kvm", - "image": "img-sve8d2fj", + "image": "img-e2vxagq6", "zone": "pek3" }, "instance_class": {{cluster.node_super_perf.instance_class}}, @@ -247,7 +250,7 @@ }, "upgrade": { "order": 1, - "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' || (exit 151); fi", + "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' && test $(netstat -nltp | grep sshd | awk '{print $4}'|awk -F : '{print $2}') -eq 22 || (exit 151); fi", "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh", "timeout": 10800 }, @@ -267,7 +270,7 @@ "role": "node_gpu", "container": { "type": "kvm", - "image": "img-kirp1glw", + "image": "img-q0kubgen", "zone": "pek3" }, "instance_class": {{cluster.node_gpu.instance_class}}, @@ -311,7 +314,7 @@ }, "upgrade": { "order": 1, - "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' || (exit 151); fi", + "pre_check": "if test $(df -m --output=avail /data | sed 1d) -lt 51200; then (exit 150); else appctl checkNodeStats '$2~/^Ready/' && test $(netstat -nltp | grep sshd | awk '{print $4}'|awk -F : '{print $2}') -eq 22 || (exit 151); fi", "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh", "timeout": 10800 }, @@ -328,7 +331,7 @@ "role": "client", "container": { "type": "kvm", - "image": "img-swxveyde", + "image": "img-frbcv58s", "zone": "pek3" }, "instance_class": {{cluster.client.instance_class}}, @@ -351,6 +354,7 @@ }, "upgrade": { "order": 2, + "pre_check": "if test $(netstat -nltp | grep sshd | awk '{print $4}'|awk -F : '{print $2}') -ne 22 ; then (exit 151);fi", "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh", "timeout": 43200 }, diff --git a/app/config.json b/app/config.json index 4f6f73c5..ace29687 100644 --- a/app/config.json +++ b/app/config.json @@ -116,6 +116,14 @@ "range": [8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, 131072, 196608, 262144], "required": "yes", "resource_group": [8192, 8192, 16384, 16384] + }, { + "key": "volume_class", + "label": "volume class", + "description": "The volume class", + "type": "integer", + "default": 6, + "range": [6], + "required": "yes" }, { "key": "volume_size", "label": "volume size", diff --git a/app/locale/zh-cn.json b/app/locale/zh-cn.json index 35bab189..af0ef3f1 100644 --- a/app/locale/zh-cn.json +++ b/app/locale/zh-cn.json @@ -12,6 +12,8 @@ "count": "数量", "Number of master for the cluster to create": "主节点数量", "resource type": "资源类型", + "volume class": "持久存储卷类型", + "The volume class": "持久存储卷类型", "volume size": "硬盘大小", "The volume size for each instance": "每个机器的硬盘大小", "load balancer": "负载均衡器", diff --git a/app/replace_policy.json b/app/replace_policy.json index ba6e71f7..8e0a213a 100644 --- a/app/replace_policy.json +++ b/app/replace_policy.json @@ -21,42 +21,16 @@ "dst": 3 }] }, - "pek3b": { - "volume_class": [{ - "src": "6", - "dst": 5 - }] - }, - "pek3c": { - "volume_class": [{ - "src": "6", - "dst": 5 - }] - }, - "sh1a": { - "volume_class": [{ - "src": "6", - "dst": 5 - }] - }, "gd2a": { "instance_class": [{ "src": "202", "dst": 201 - }], - "volume_class": [{ - "src": "6", - "dst": 5 }] }, "gd2b": { "instance_class": [{ "src": "202", "dst": 201 - }], - "volume_class": [{ - "src": "6", - "dst": 5 }] }, "ap2a": {