Upgrade to zk 3.4.14

QingCloudAppcenter · Feb 29, 2020 · 44fbfa2 · 44fbfa2
1 parent 1ff0453
commit 44fbfa2
Show file tree

Hide file tree

Showing 38 changed files with 692 additions and 256 deletions.
diff --git a/ansible/.gitignore b/ansible/.gitignore
@@ -0,0 +1,3 @@
+**/files/tmp/
+hosts
+make.retry
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
@@ -0,0 +1 @@
+zk_version: 3.4.14
diff --git a/ansible/make.yml b/ansible/make.yml
@@ -0,0 +1,22 @@
+---
+- name: zk tasks
+  hosts: all
+  vars:
+    target_env: "{{ lookup('env', 'target') }}"
+  tasks:
+  - include_role:
+      name: "{{ name }}"
+    loop:
+    - update-apt-sources-1.0.0
+    - disable-apt-jobs-1.0.0
+    - disable-motd-1.0.0
+    - app-agent-1.0.1
+    - appctl-1.1.0
+    - arping-1.0.0
+    - caddy-1.0.6
+    - java-1.0.1
+    - jq-1.0.3
+    - netcat-openbsd-1.0.0
+    - zookeeper
+    loop_control:
+      loop_var: name
diff --git a/ansible/requirements.yml b/ansible/requirements.yml
@@ -0,0 +1,12 @@
+- src: https://qingcloudappcenter.github.io/ansible-roles/app-agent-1.0.1.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/appctl-1.1.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/caddy-1.0.6.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.0.2.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/install-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/java-1.0.1.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/jq-1.0.3.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/netcat-openbsd-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/update-apt-sources-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/disable-apt-jobs-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/disable-motd-1.0.0.tar.gz
diff --git a/ansible/roles/zookeeper/files/etc/confd/conf.d/caddy.sh.toml b/ansible/roles/zookeeper/files/etc/confd/conf.d/caddy.sh.toml
@@ -0,0 +1,8 @@
+[template]
+src = "caddy.sh.tmpl"
+dest = "/opt/app/bin/tmpl/caddy.sh"
+mode = "0700"
+keys = [
+  "/",
+]
+reload_cmd = "/opt/app/bin/tmpl/caddy.sh; appctl reload caddy"
diff --git a/ansible/roles/zookeeper/files/etc/confd/conf.d/common.sh.toml b/ansible/roles/zookeeper/files/etc/confd/conf.d/common.sh.toml
diff --git a/ansible/roles/zookeeper/files/etc/confd/conf.d/zkrest.sh.toml b/ansible/roles/zookeeper/files/etc/confd/conf.d/zkrest.sh.toml
@@ -0,0 +1,8 @@
+[template]
+src = "zkrest.sh.tmpl"
+dest = "/opt/app/bin/tmpl/zkrest.sh"
+mode = "0700"
+keys = [
+  "/",
+]
+reload_cmd = "/opt/app/bin/tmpl/zkrest.sh; appctl reload zkrest"
diff --git a/ansible/roles/zookeeper/files/etc/confd/conf.d/zoo.cfg.toml b/ansible/roles/zookeeper/files/etc/confd/conf.d/zoo.cfg.toml
diff --git a/ansible/roles/zookeeper/files/etc/confd/conf.d/zookeeper.sh.toml b/ansible/roles/zookeeper/files/etc/confd/conf.d/zookeeper.sh.toml
@@ -0,0 +1,8 @@
+[template]
+src = "zookeeper.sh.tmpl"
+dest = "/opt/app/bin/tmpl/zookeeper.sh"
+mode = "0700"
+keys = [
+  "/",
+]
+reload_cmd = "/opt/app/bin/tmpl/zookeeper.sh; appctl reload zookeeper"
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/caddy.sh/01.svc-caddy.env.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/caddy.sh/01.svc-caddy.env.tmpl
@@ -0,0 +1,3 @@
+flush /opt/app/bin/envs/svc-caddy.env << CADDY_ENV_EOF
+SERVICES="\$SERVICES caddy/{{ getv "/env/web.console.enabled" "true" }}/http:80"
+CADDY_ENV_EOF
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/caddy.sh/02.caddyfile.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/caddy.sh/02.caddyfile.tmpl
@@ -0,0 +1,13 @@
+flush /opt/app/conf/caddy/caddyfile << CADDYFILE_EOF
+{{ getv "/host/ip" }}:80 {
+  {{- with getv "/env/web.console.password" "" }}
+  basicauth / "{{ getv "/env/web.console.username" "admin" }}" "{{ . }}"
+  {{- end }}
+  root /data
+  gzip
+  browse /zookeeper/dump
+  browse /zookeeper/logs
+  browse /zkrest/logs
+  tls off
+}
+CADDYFILE_EOF
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/caddy.sh/03.index.html.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/caddy.sh/03.index.html.tmpl
@@ -0,0 +1,92 @@
+flush /opt/app/conf/caddy/index.html << HTML_EOF
+<!DOCTYPE html>
+<html>
+  <head>
+    <style>
+      html, body, ul {
+        margin: 0;
+        padding: 0;
+      }
+
+      header {
+        background: #f2f2f2;
+        font-family: sans-serif;
+        font-size: 1.2em;
+        height: 3.2em;
+        line-height: 3.2em;
+        padding-left: 5%;
+      }
+
+      .meta {
+        border-bottom: 1px solid #9c9c9c;
+        font-family: Verdana; sans-serif;
+        font-size: 12px;
+        line-height: 44px;
+        padding-left: 5%;
+      }
+
+      .listing {
+        list-style-type: none;
+      }
+
+      .listing>li {
+        border-bottom: 1px solid #dadada;
+        font-family: sans-serif;
+        font-size: 14px;
+        line-height: 36px;
+        padding-left: 5%;
+      }
+
+      .listing>li.header {
+        font-size: 16px;
+        font-weight: bold;
+        line-height: 46px;
+      }
+
+      .listing>li>span {
+        margin-right: 1em;
+      }
+
+      .listing>li>span.sub-1 {
+        margin-left: 1em;
+      }
+
+      .listing>li>a {
+        color: #006ed3;
+      }
+
+      footer {
+        font-size: 12px;
+        line-height: 72px;
+        text-align: center;
+      }
+    </style>
+    <title>Files</title>
+  </head>
+  <body>
+    <header>文件查看器</header>
+    <main>
+      {{- $zkNodes := getvs "/hosts/*/ip" }}
+      <div class="meta">
+        <span>共 <b>{{ len $zkNodes }}</b> 个 ZooKeeper 节点</span>
+      </div>
+      <ul class="listing">
+        <li class="header">节点/目录</li>
+        {{- range $zkNodes }}
+        {{- if eq . (getv "/host/ip") }}
+        <li><span>⊟</span>{{ . }}</li>
+        <li><span class="sub-1">⊞</span><a href="./zookeeper/dump/">zookeeper/dump/</a></li>
+        <li><span class="sub-1">⊞</span><a href="./zookeeper/logs/">zookeeper/logs/</a></li>
+        <li><span class="sub-1">⊞</span><a href="./zkrest/logs/">zkrest/logs/</a></li>
+        {{- else }}
+        <li><span>⊞</span><a href="//{{ . }}/">{{ . }}</a></li>
+        {{- end }}
+        {{- end }}
+      </ul>
+    </main>
+    <footer>
+      Served with <a rel="noopener noreferrer" href="https://caddyserver.com">Caddy</a>
+    </footer>
+  </body>
+</html>
+HTML_EOF
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/common.sh.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/common.sh.tmpl
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/zkrest.sh/01.svc-zkrest.env.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/zkrest.sh/01.svc-zkrest.env.tmpl
@@ -0,0 +1,3 @@
+flush /opt/app/bin/envs/svc-zkreset.env << ZKREST_ENV_EOF
+SERVICES="\$SERVICES zkrest/{{ getv "/env/zkrest.enabled" "true" }}/http:9998"
+ZKREST_ENV_EOF
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/zoo.cfg.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/zoo.cfg.tmpl
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/zookeeper.sh/01.node.env.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/zookeeper.sh/01.node.env.tmpl
@@ -0,0 +1,34 @@
+# global var which will be used in other tmpl files
+{{- $usingGsid := split "appv-9b7na511 appv-89xrqcih" " " | filter (getv "/upgrade-audit/from_app_version" "none") }}
+{{- $allNodes := lsdir "/hosts" }}
+{{- $joiningNodes := lsdir "/adding-hosts" }}
+{{- $leavingNodes := lsdir "/deleting-hosts" }}
+
+allNodes="$(sort -V << ALL_NODES_EOF
+{{- range $instanceId := $allNodes }}
+{{- if $joiningNodes | filter $instanceId }}
+joining/
+{{- else if $leavingNodes | filter $instanceId }}
+leaving/
+{{- else }}
+stable/
+{{- end }}
+{{- getv (printf "/hosts/%s/sid" $instanceId) }}/
+{{- getv (printf "/hosts/%s/node_id" $instanceId) }}/
+{{- getv (printf "/hosts/%s/ip" $instanceId) }}
+{{- end }}
+ALL_NODES_EOF
+)"
+joiningNodes="$(echo "$allNodes" | awk -F/ '$1~/^j/ {print $2"/"$3"/"$4}' | xargs)"
+
+flush /opt/app/bin/envs/node-zookeeper.env << ZK_ENV_EOF
+SERVICES="\$SERVICES zookeeper/true/tcp:{{ getv "/cluster/endpoints/client/port" "2181" }}"
+NODE_CTL="zookeeper"
+DATA_MOUNTS=/data
+IS_UPGRADING={{ exists "/upgrade-audit/from_app_version" }}
+MY_IP={{ getv "/host/ip" }}
+MY_ZK_ID={{ if $usingGsid }}{{ getv "/host/gsid" }}{{ else }}{{ getv "/host/sid" }}{{ end }}
+JOINING_NODES="$joiningNodes"
+LEAVING_NODES="$(echo "$allNodes" | awk -F/ '$1~/^l/ {print $2"/"$3"/"$4}' | xargs)"
+STABLE_NODES="$(echo "$allNodes" | awk -F/ '$1~/^s/ {print $2"/"$3"/"$4}' | xargs)"
+ZK_ENV_EOF
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/zookeeper.sh/02.java.env.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/zookeeper.sh/02.java.env.tmpl
@@ -0,0 +1,16 @@
+dumpFlags="-XX:HeapDumpPath=/data/zookeeper/dump/dump.hprof"
+{{- if eq "true" (getv "/env/zkAdminEnabled" "false") }}
+pushd /opt/zookeeper/current
+jarFiles="$(echo -n lib/slf4j-api*):zookeeper-$ZK_VERSION.jar"
+authFlags="-Dzookeeper.DigestAuthenticationProvider.superDigest=$(java -cp $jarFiles \
+           org.apache.zookeeper.server.auth.DigestAuthenticationProvider \
+           "{{ getv "/env/zkAdminUsername" "" }}:{{ getv "/env/zkAdminPassword" "" }}" | awk -F'->' '{print $2}')"
+popd
+{{- end }}
+
+roFlags="-Dreadonlymode.enabled={{ getv "/env/readonlymode.enabled" "false" }}"
+
+flush /opt/app/conf/zookeeper/java.env << ZK_JAVA_ENV_EOF
+SERVER_JVMFLAGS="$dumpFlags $authFlags $roFlags"
+ZK_SERVER_HEAP={{ sub (getv "/host/memory") 90 }}
+ZK_JAVA_ENV_EOF
diff --git a/ansible/roles/zookeeper/files/etc/confd/templates/zookeeper.sh/03.zoo.cfg.tmpl b/ansible/roles/zookeeper/files/etc/confd/templates/zookeeper.sh/03.zoo.cfg.tmpl
@@ -0,0 +1,34 @@
+firstJoiningNode="${joiningNodes%% *}"
+
+flush /opt/app/conf/zookeeper/zoo.cfg << ZOO_CONF_EOF
+tickTime={{getv "/env/tickTime" "2000"}}
+initLimit={{getv "/env/initLimit" "10"}}
+syncLimit={{getv "/env/syncLimit" "5"}}
+dataDir=/data/zookeeper
+clientPort={{getv "/cluster/endpoints/client/port" "2181"}}
+maxClientCnxns={{getv "/env/maxClientCnxns" "1000"}}
+autopurge.snapRetainCount={{getv "/env/autopurge.snapRetainCount" "3"}}
+autopurge.purgeInterval={{getv "/env/autopurge.purgeInterval" "0"}}
+4lw.commands.whitelist=mntr,{{ getv "/env/4lw.commands.whitelist" "" }}
+tcpKeepAlive={{ getv "/env/tcpKeepAlive" "false" }}
+
+{{- range $allNodes }}
+{{- if not ($joiningNodes | filter .) }}
+server.{{ if $usingGsid }}{{ getv (printf "/hosts/%s/gsid" .) }}{{ else }}{{ getv (printf "/hosts/%s/sid" .) }}{{ end }}={{ getv (printf "/hosts/%s/ip" .) }}:2888:3888
+{{- end }}
+{{- end }}
+
+{{- $mySid := getv "/host/sid" }}
+{{- range $joiningNodes }}
+{{- $sid := getv (printf "/adding-hosts/%s/sid" .) }}
+{{- if le $sid $mySid }}
+server.{{ if $usingGsid }}{{ getv (printf "/adding-hosts/%s/gsid" .) }}{{ else }}{{ $sid }}{{ end }}={{ getv (printf "/adding-hosts/%s/ip" .) }}:2888:3888
+{{- end }}
+{{- end }}
+
+{{- if $joiningNodes }}
+{{- if and (not ($joiningNodes | filter (getv "/host/instance_id"))) (eq (sub (len $allNodes) (len $joiningNodes)) 1) }}
+server.${firstJoiningNode%%/*}=${firstJoiningNode##*/}:2888:3888
+{{- end }}
+{{- end }}
+ZOO_CONF_EOF
diff --git a/ansible/roles/zookeeper/files/lib/systemd/system/post-docker-start.service b/ansible/roles/zookeeper/files/lib/systemd/system/post-docker-start.service
diff --git a/ansible/roles/zookeeper/files/lib/systemd/system/zkrest.service b/ansible/roles/zookeeper/files/lib/systemd/system/zkrest.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=ZooKeeper REST Gateway
+Documentation=https://zookeeper.apache.org
+Requires=network.target
+After=network.target
+
+[Service]
+Type=simple
+User=zookeeper
+Group=zookeeper
+ExecStart=/opt/zkrest/current/bin/zkRest.sh
+WorkingDirectory=/data/zkrest
+
+[Install]
+WantedBy=default.target