add precheck process for upgrade (#290)

* add precheck process for upgrade * add escape * fix return in mustache file * fix command in mustache * update translate * del space * update translate * add regex * update translation
QingCloudAppcenter · Sep 22, 2020 · 12b7b19 · 12b7b19
1 parent c42b8d1
commit 12b7b19
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 6 deletions.
diff --git a/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s.sh b/ansible/roles/app-role-k8s/files/opt/app/current/bin/node/k8s.sh
@@ -376,11 +376,11 @@ waitAllNodesUpgraded() {
 }
 
 waitAllMasterNodesUpgradedAndReady() {
-  retry 3600 2 0 checkNodeStats '$2=="Ready"&&$3~/master/&&$5=="v'$K8S_VERSION'"' $STABLE_MASTER_NODES $JOINING_MASTER_NODES
+  retry 3600 2 0 checkNodeStats '$2~/^Ready/&&$3~/master/&&$5=="v'$K8S_VERSION'"' $STABLE_MASTER_NODES $JOINING_MASTER_NODES
 }
 
 waitAllNodesUpgradedAndReady() {
-  retry 600 1 0 checkNodeStats '$2=="Ready"&&$5=="v'$K8S_VERSION'"'
+  retry 600 1 0 checkNodeStats '$2~/^Ready/&&$5=="v'$K8S_VERSION'"'
 }
 
 waitAllMasterNodesJoined(){

diff --git a/app/cluster.json.mustache b/app/cluster.json.mustache
@@ -119,7 +119,7 @@
       },
       "upgrade": {
         "order": 1,
-        "pre_check": "bash -c 'if test $(df -m --output=avail /data | sed 1d) -lt 51200; then exit 150; fi'",
+        "pre_check": "bash -c 'if test $(df -m --output=avail /data | sed 1d) -lt 51200; then exit 150; fi' && appctl checkNodeStats '$2~/^Ready/' || bash -c 'exit 151'",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 10800
       },
@@ -190,7 +190,7 @@
       },
       "upgrade": {
         "order": 1,
-        "pre_check": "bash -c 'if test $(df -m --output=avail /data | sed 1d) -lt 51200; then exit 150; fi'",
+        "pre_check": "bash -c 'if test $(df -m --output=avail /data | sed 1d) -lt 51200; then exit 150; fi' && appctl checkNodeStats '$2~/^Ready/' || bash -c 'exit 151'",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 10800
       },
@@ -252,7 +252,7 @@
       },
       "upgrade": {
         "order": 1,
-        "pre_check": "bash -c 'if test $(df -m --output=avail /data | sed 1d) -lt 51200; then exit 150; fi'",
+        "pre_check": "bash -c 'if test $(df -m --output=avail /data | sed 1d) -lt 51200; then exit 150; fi' && appctl checkNodeStats '$2~/^Ready/' || bash -c 'exit 151'",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 10800
       },
@@ -316,7 +316,7 @@
       },
       "upgrade": {
         "order": 1,
-        "pre_check": "bash -c 'if test $(df -m --output=avail /data | sed 1d) -lt 51200; then exit 150; fi'",
+        "pre_check": "bash -c 'if test $(df -m --output=avail /data | sed 1d) -lt 51200; then exit 150; fi' && appctl checkNodeStats '$2~/^Ready/' || bash -c 'exit 151'",
         "cmd": "/upgrade/vm-files/opt/app/current/bin/node/upgrade.sh",
         "timeout": 10800
       },

diff --git a/app/locale/zh-cn.json b/app/locale/zh-cn.json
@@ -107,6 +107,7 @@
     "err_code136": "迁移待删除节点上的工作负载时出错，已尝试回退此节点的状态但执行 kubectl uncordon 时出错，请检查主节点的 kube-apiserver 服务是否正常，可以在服务稳定后重新尝试删除节点，如需协助请工单联系",
     "err_code137": "请勿删除主节点",
     "err_code150": "检测到某些节点没有足够的可用磁盘空间，请确保所有 k8s 节点至少有 50G 空闲磁盘空间再尝试升级",
+    "err_code151": "检测到某些节点为非就绪（Ready）状态，请确保所有节点均包含就绪（Ready）状态后再升级；节点状态可通过 kubectl get nodes 查看",
     "network plugin": "网卡插件",
     "Choose a network plugin": "选择网卡插件",
     "SSD node label": "企业型节点标记",