Merge pull request #409 from zryfish/bump_kubesphere_to_v3.1

bump kubesphere to v3.1

ansible/group_vars/all.yml

@@ -1,25 +1,24 @@
-app_version: 3.0.1
+app_version: 3.1.0
 gcr_mirror: kubesphere
-etcd_version: 3.3.12
+etcd_version: 3.4.13
 cni_version: 0.8.6
-k8s_version: 1.18.12
+k8s_version: 1.19.8
 flannel_version: 0.12.0
 coredns_version: 1.6.9
-calico_version: 3.15.1
-hostnic_version: 1.0.0
-qingcloud_csi_version: 1.2.6
-qingcloud_ccm_version: 1.4.5
-ks_version: 3.0.0
-ks_installer_image_tag: qke-3.0.1
+calico_version: 3.16.3
+hostnic_version: 1.0.0-alpha.5
+qingcloud_csi_version: 1.2.7
+qingcloud_ccm_version: 1.4.7
+ks_version: 3.1.0
+ks_installer_image_tag: v3.1.0
 helm_version: 3.2.1
 helm_2to3_version: 0.8.0
 helm_stable_repo: https://charts.kubesphere.io/mirror
 helm_main_repo: https://charts.kubesphere.io/main
 helm_test_repo: https://charts.kubesphere.io/test
-nodelocaldns_version: 1.15.5
-nvidia_tesla_version: 418.116.00
+nodelocaldns_version: 1.15.12
+nvidia_tesla_version: 460.73.01
 nvidia_plugin_version: 1.0.0-beta4
-# 本地 images package 的位置
 local_images_path: /images
 ks_images_package: allks.tgz
 k8s_images_package: allk8s.tgz
@@ -33,51 +32,6 @@ binaries:
 - /opt/k8s
 
 docker_images_k8s:
-- kubesphere/cloud-controller-manager:v1.4.4
-
-docker_images_ks:
-- alpine:3.10.4
-- docker:19.03
-- haproxy:2.0.4
-- jenkins/jenkins:2.176.2
-- kubesphere/builder-base:v2.1.0
-- kubesphere/builder-go:v2.1.0
-- kubesphere/builder-maven:v2.1.0
-- kubesphere/builder-nodejs:v2.1.0
-- kubesphere/elasticsearch-oss:6.7.0-1
-- kubesphere/etcd:v3.2.18
-- kubesphere/java-11-centos7:v2.1.0
-- kubesphere/java-11-runtime:v2.1.0
-- kubesphere/java-8-centos7:v2.1.0
-- kubesphere/java-8-runtime:v2.1.0
-- kubesphere/kubectl:v1.0.0
-- kubesphere/kube-rbac-proxy:v0.4.1
-- kubesphere/nodejs-4-centos7:v2.1.0
-- kubesphere/nodejs-6-centos7:v2.1.0
-- kubesphere/nodejs-8-centos7:v2.1.0
-- kubesphere/python-27-centos7:v2.1.0
-- kubesphere/python-34-centos7:v2.1.0
-- kubesphere/python-35-centos7:v2.1.0
-- kubesphere/python-36-centos7:v2.1.0
-- kubesphere/s2i-binary:v2.1.0
-- kubesphere/s2ioperator:v2.1.1
-- kubesphere/s2irun:v2.1.1
-- kubesphere/tomcat85-java11-centos7:v2.1.0
-- kubesphere/tomcat85-java11-runtime:v2.1.0
-- kubesphere/tomcat85-java8-centos7:v2.1.0
-- kubesphere/tomcat85-java8-runtime:v2.1.0
-- minio/mc:RELEASE.2019-08-07T23-14-43Z
-- minio/minio:RELEASE.2019-08-07T01-59-21Z
-- mysql:8.0.11
-- nginx:1.14-alpine
-- osixia/openldap:1.3.0
-- redis:5.0.5-alpine
-
-docker_images_k8s_new:
-- calico/cni:v3.15.1
-- calico/kube-controllers:v3.15.1
-- calico/node:v3.15.1
-- calico/pod2daemon-flexvol:v3.15.1
 - coredns/coredns:1.6.9
 - csiplugin/csi-attacher:v2.1.1
 - csiplugin/csi-node-driver-registrar:v1.2.0
@@ -86,66 +40,122 @@ docker_images_k8s_new:
 - csiplugin/csi-resizer:v0.4.0
 - csiplugin/csi-snapshotter:v2.0.1
 - csiplugin/snapshot-controller:v2.0.1
-- kubesphere/hostnic:v1.0.0
 - nvidia/k8s-device-plugin:1.0.0-beta4
 - kubesphere/flannel:v0.12.0
-- kubesphere/kube-apiserver:v1.18.12
-- kubesphere/kube-controller-manager:v1.18.12
-- kubesphere/kube-proxy:v1.18.12
-- kubesphere/kube-scheduler:v1.18.12
-- kubesphere/pause:3.2
+- kubesphere/pause:3.1
 
-docker_images_ks_new:
-- istio/citadel:1.4.8
-- istio/galley:1.4.8
-- istio/kubectl:1.4.8
-- istio/mixer:1.4.8
-- istio/pilot:1.4.8
-- istio/proxyv2:1.4.8
-- istio/sidecar_injector:1.4.8
+docker_images_ks:
 - jaegertracing/jaeger-agent:1.17
 - jaegertracing/jaeger-collector:1.17
 - jaegertracing/jaeger-es-index-cleaner:1.17.1
 - jaegertracing/jaeger-operator:1.17.1
 - jaegertracing/jaeger-query:1.17
 - jenkins/jnlp-slave:3.27-1
 - jimmidyson/configmap-reload:v0.3.0
-- kubesphere/alert-adapter:v3.0.0
-- kubesphere/alerting-dbinit:v3.0.0
-- kubesphere/alerting:v2.1.2
 - kubesphere/fluentbit-operator:migrator
-- kubesphere/fluentbit-operator:v0.2.0
-- kubesphere/fluent-bit:v1.4.6
-- kubesphere/jenkins-uc:v3.0.0
-- kubesphere/ks-apiserver:v3.0.0
-- kubesphere/ks-console:v3.0.0
-- kubesphere/ks-controller-manager:v3.0.0
-- kubesphere/ks-devops:flyway-v3.0.0
-- kubesphere/ks-installer:qke-3.0.0
-- kubesphere/ks-upgrade:v3.0.0
 - kubesphere/kube-auditing-operator:v0.1.2
 - kubesphere/kube-auditing-webhook:v0.1.2
 - kubesphere/kube-events-exporter:v0.1.0
 - kubesphere/kube-events-operator:v0.1.0
-- kubesphere/kube-events-ruler:v0.1.0
-- kubesphere/kubefed:v0.3.0
-- kubesphere/kube-state-metrics:v1.9.6
 - kubesphere/log-sidecar-injector:1.1
 - kubesphere/elasticsearch-curator:v5.7.6
-- kubesphere/metrics-server:v0.3.7
 - kubesphere/nginx-ingress-controller:0.24.1
 - kubesphere/node-exporter:ks-v0.18.1
-- kubesphere/notification:flyway_v2.1.2
-- kubesphere/notification-manager-operator:v0.1.0
-- kubesphere/notification-manager:v0.1.0
-- kubesphere/notification:v2.1.2
-- kubesphere/prometheus-config-reloader:v0.38.3
-- kubesphere/prometheus-operator:v0.38.3
-- kubesphere/tower:v0.1.0
 - mirrorgooglecontainers/defaultbackend-amd64:1.4
-- openpitrix/generate-kubeconfig:v0.5.0
-- openpitrix/openpitrix:flyway-v0.5.0
-- openpitrix/openpitrix:v0.5.0
-- openpitrix/release-app:sha-303629d
 - prom/alertmanager:v0.21.0
-- prom/prometheus:v2.20.1
+- docker:19.03
+
+docker_images_k8s_new:
+- kubesphere/kube-apiserver:v1.19.8
+- kubesphere/kube-scheduler:v1.19.8
+- kubesphere/kube-proxy:v1.19.8
+- kubesphere/kube-controller-manager:v1.19.8
+- kubesphere/pause:3.2
+- kubesphere/etcd:v3.4.13
+- calico/cni:v3.16.3
+- calico/kube-controllers:v3.16.3
+- calico/node:v3.16.3
+- calico/pod2daemon-flexvol:v3.16.3
+- kubesphere/k8s-dns-node-cache:1.15.12
+- kubesphere/nfs-client-provisioner:v3.1.0-k8s1.11
+- csiplugin/csi-qingcloud:v1.2.1
+- csiplugin/csi-neonsan:v1.2.1
+- csiplugin/csi-neonsan-ubuntu:v1.2.1
+- csiplugin/csi-neonsan-centos:v1.2.1
+- kubesphere/cloud-controller-manager:v1.4.7
+- kubesphere/hostnic:v1.0.0-alpha.5
+- kubesphere/metrics-server:v0.4.2
+
+docker_images_ks_new:
+- kubesphere/ks-apiserver:v3.1.0
+- kubesphere/ks-console:v3.1.0
+- kubesphere/ks-controller-manager:v3.1.0
+- kubesphere/ks-installer:v3.1.0
+- kubesphere/kubectl:v1.19.0
+- kubesphere/nginx-ingress-controller:v0.35.0
+- kubesphere/kubefed:v0.7.0
+- kubesphere/tower:v0.2.0
+- kubesphere/prometheus-config-reloader:v0.42.1
+- kubesphere/prometheus-operator:v0.42.1
+- kubesphere/kube-state-metrics:v1.9.7
+- prom/prometheus:v2.26.0
+- prom/node-exporter:v0.18.1
+- kubesphere/ks-alerting-migration:v3.1.0
+- kubesphere/notification-manager-operator:v1.0.0
+- kubesphere/notification-manager:v1.0.0
+- kubesphere/kube-rbac-proxy:v0.8.0
+- openebs/provisioner-localpv:2.3.0
+- thanosio/thanos:v0.18.0
+- grafana/grafana:7.4.3
+- kubesphere/fluentbit-operator:v0.5.0
+- kubesphere/fluent-bit:v1.6.9
+- elastic/filebeat:6.7.0
+- kubesphere/kube-events-ruler:v0.2.0
+- istio/pilot:1.6.10
+- istio/proxyv2:1.6.10
+- kubesphere/kiali:v1.26.1
+- kubesphere/kiali-operator:v1.26.1
+- kubesphere/ks-jenkins:2.249.1
+- kubesphere/s2ioperator:v3.1.0
+- kubesphere/openpitrix-jobs:v3.1.0
+- weaveworks/scope:1.13.0
+- kubeedge/cloudcore:v1.6.1
+- kubesphere/edge-watcher:v0.1.0
+- kubesphere/kube-rbac-proxy:v0.5.0
+- kubesphere/edge-watcher-agent:v0.1.0
+- kubesphere/examples-bookinfo-productpage-v1:1.16.2
+- kubesphere/examples-bookinfo-reviews-v1:1.16.2
+- kubesphere/examples-bookinfo-reviews-v2:1.16.2
+- kubesphere/examples-bookinfo-reviews-v3:1.16.2
+- kubesphere/examples-bookinfo-details-v1:1.16.2
+- kubesphere/examples-bookinfo-ratings-v1:1.16.3
+- busybox:1.31.1
+- joosthofman/wget:1.0
+- kubesphere/netshoot:v1.0
+- wordpress:4.8-apache
+- mirrorgooglecontainers/hpa-example:latest
+- java:openjdk-8-jre-alpine
+- fluent/fluentd:v1.4.2-2.0
+- perl:latest
+- osixia/openldap:1.3.0
+- redis:5.0.5-alpine
+- alpine:3.10.4
+- haproxy:2.0.4
+- nginx:1.14-alpine
+- minio/minio:RELEASE.2019-08-07T01-59-21Z
+- minio/mc:RELEASE.2019-08-07T23-14-43Z
+- kubesphere/elasticsearch-oss:6.7.0-1
+- kubesphere/s2irun:v2.1.1
+- kubesphere/builder-base:v3.1.0
+- kubesphere/builder-nodejs:v3.1.0
+- kubesphere/builder-maven:v3.1.0
+- kubesphere/builder-go:v3.1.0
+- kubesphere/s2i-binary:v2.1.0
+- kubesphere/tomcat85-java11-centos7:v2.1.0
+- kubesphere/tomcat85-java11-runtime:v2.1.0
+- kubesphere/tomcat85-java8-centos7:v2.1.0
+- kubesphere/tomcat85-java8-runtime:v2.1.0
+- kubesphere/java-11-centos7:v2.1.0
+- kubesphere/java-8-centos7:v2.1.0
+- kubesphere/java-8-runtime:v2.1.0
+- kubesphere/java-11-runtime:v2.1.0

ansible/make.yml

@@ -4,7 +4,7 @@
   vars:
     target_env: "{{ lookup('env', 'target') }}"
   roles:
-  - docker-1.0.8
+  - docker-1.0.10
   - docker-images
 
 - hosts: k8s-client,k8s-node,gpu-node
@@ -33,8 +33,8 @@
   vars:
     target_env: "{{ lookup('env', 'target') }}"
   roles:
-  - qingcloud-cli-1.0.6
-  - docker-1.0.8
+  - qingcloud-cli-1.0.7
+  - docker-1.0.10
   - etcd-1.1.0
   - k8s-node
   # put extra modules above

ansible/requirements.yml

@@ -3,7 +3,7 @@
 - src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.5.tar.gz
 - src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.1.0.tar.gz
 - src: https://qingcloudappcenter.github.io/ansible-roles/create-service-user-1.0.0.tar.gz
-- src: https://qingcloudappcenter.github.io/ansible-roles/docker-1.0.8.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/docker-1.0.10.tar.gz
 - src: https://qingcloudappcenter.github.io/ansible-roles/golang-1.0.3.tar.gz
 - src: https://qingcloudappcenter.github.io/ansible-roles/etcd-1.1.0.tar.gz
 - src: https://qingcloudappcenter.github.io/ansible-roles/install-1.0.6.tar.gz
@@ -12,4 +12,4 @@
 - src: https://qingcloudappcenter.github.io/ansible-roles/disable-apt-jobs-1.0.0.tar.gz
 - src: https://qingcloudappcenter.github.io/ansible-roles/disable-motd-1.0.0.tar.gz
 - src: https://qingcloudappcenter.github.io/ansible-roles/update-apt-sources-1.0.0.tar.gz
-- src: https://qingcloudappcenter.github.io/ansible-roles/qingcloud-cli-1.0.6.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/qingcloud-cli-1.0.7.tar.gz

ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl

@@ -22,6 +22,6 @@ LB_IP_FROM_V1={{ $lbIpFromV1 }}
 KS_MODULES_COUNT=10
 {{- else }}
 {{- $extraModules := getvs "/env/extra.modules" }}
-KS_MODULES_COUNT={{ add 2 (len (split (join $extraModules "") "," | filter "ks-*")) }}
+KS_MODULES_COUNT={{ add 3 (len (split (join $extraModules "") "," | filter "ks-*")) }}
 {{- end }}
 NODE_ENV_EOF

ansible/roles/app-role-client/files/opt/app/current/bin/node/client.sh

@@ -87,9 +87,11 @@ checkKsInstallerDone() {
   local output; output="$(runKubectl -n kubesphere-system logs --tail 50 $podName)" || return $EC_KS_INSTALL_LOGS_ERR
   if echo "$output" | grep "^PLAY RECAP **" -A1 | egrep -o "failed=[1-9]"; then return $EC_KS_INSTALL_FAILED; fi
   echo "$output" | grep -oF 'Welcome to KubeSphere!' || return $EC_KS_INSTALL_RUNNING
-  local endStrings="total: $KS_MODULES_COUNT     completed:$KS_MODULES_COUNT"
+  #local endStrings="is successful  ($KS_MODULES_COUNT/$KS_MODULES_COUNT)"
   if $IS_UPGRADING_FROM_V2; then endStrings=" failed=0 "; fi
-  echo "$output" | grep "Welcome to KubeSphere!" -B4 | grep -oF "$endStrings" || return $EC_KS_INSTALL_DONE_WITH_ERR
+  # if tail of installer log has line like "task openpitrix status is failed", means one or more components are failed
+  # to install.
+  !(echo "$output" | grep "Welcome to KubeSphere!" -B30 | grep -q "^task.*failed") || return $EC_KS_INSTALL_DONE_WITH_ERR
 }
 
 getKsInstallerPodName() {

ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/01.node.env.tmpl

@@ -44,6 +44,7 @@ ALL_NODES_EOF
 myRole={{ getv "/host/role" }}
 mySid={{ getv "/host/sid" }}
 myInstanceId={{ getv "/host/instance_id" }}
+myZone={{ getv "/host/zone" }}
 myNodeName=$(buildNodeName $myRole $mySid)
 
 flush /opt/app/current/bin/envs/node.env << NODE_ENV_EOF
@@ -53,10 +54,12 @@ CLUSTER_TAG={{ join (getvs "/cluster/cluster_tag") "" }}
 CLUSTER_VXNET={{ getv "/cluster/vxnet" }}
 CLUSTER_ZONE={{ getv "/cluster/zone" }}
 CLUSTER_API_SERVER={{ getv "/cluster/api_server/host" "api.qingcloud.com" }}
+{{- if exists "/cluster/region_id" }}CLUSTER_REGION={{ getv "/cluster/region_id" }}{{- end }}
 MY_IP={{ getv "/host/ip" }}
 MY_SID=$mySid
 MY_ROLE=$myRole
 MY_NODE_NAME=$myNodeName
+MY_ZONE=$myZone
 MY_INSTANCE_ID=$myInstanceId
 UPGRADED_FROM_V1={{ $upgradedFromV1 }}
 LB_IP_FROM_V1={{ $lbIpFromV1 }}

ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/05.kubeadm.conf.tmpl

@@ -34,7 +34,9 @@ apiServer:
     advertise-address: 0.0.0.0
     {{- if $kubeAuditPolicy }}
     audit-policy-file: $KUBE_AUDIT_POLICY_RUNTIME_FILE
+    {{- if $kubeAuditWebhook }}
     audit-webhook-config-file: $KUBE_AUDIT_WEBHOOK_RUNTIME_FILE
+    {{- end }}
     audit-log-maxage: "{{ getv "/env/keep_audit_days" }}"
     audit-log-path: /etc/kubernetes/audit/logs/audit.log
     audit-log-maxsize: "{{ getv "/env/kube_audit_log_maxsize" "1" }}"

ansible/roles/app-role-k8s/files/etc/confd/templates/kube.sh/09.ks-config.dynamic.yml.tmpl

@@ -28,7 +28,8 @@ common:
 logging:
   enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "logging")) 0) }}
 openpitrix:
-  enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "openpitrix")) 0) }}
+  store:
+    enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "openpitrix")) 0) }}
 devops:
   enabled: {{ or (and $upgradedFromV1 $upgrading) (gt (len ($extraModules | filter "devops")) 0) }}
 servicemesh: