Merge pull request #5 from djangoyi-yunify/1.2.4-v1.0.0

1.2.4 v1.0.0
QingCloudAppcenter · Apr 19, 2022 · a56814e · a56814e
2 parents 1488a7f + 24d4f9a
commit a56814e
Show file tree

Hide file tree

Showing 13 changed files with 164 additions and 121 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+ansible/hosts
+ansible/files/tmp
+ansible/roles/*/files/tmp
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
@@ -1,7 +1,8 @@
 appAgentVersion: v1.0.6
-opensearch_version: 1.3.0
-package_version: 1.3.0
-opensearch_dashboards_version: 1.3.0
-opensearch_dashboards_package_version: 1.3.0
+opensearch_version: 1.2.4
+package_version: 1.2.4
+opensearch_dashboards_version: 1.2.0
+opensearch_dashboards_package_version: 1.2.0
 elk_version: 7.10.2
-logstash_version: 7.13.2
+logstash_version: 7.13.2
+local_cache_path: /root/.ansible/cache
diff --git a/ansible/make.yml b/ansible/make.yml
@@ -4,12 +4,12 @@
   vars:
     target_env: "{{ lookup('env', 'target') }}"
   vars_files:
-  - /root/.qcacode
+  #- /root/.qcacode
   strategy: free
   roles:
   - app-agent-1.0.1
   - appctl-1.0.9
-  - arping-1.0.0
+  - arping-1.0.5
   - common
   - role: caddy-1.0.6
     vars:
@@ -26,8 +26,8 @@
     - java-1.0.1
     - opensearch
     - jq-1.0.3
-    - node-opensearch
     - node-exporter-0.18.1
+    - node-opensearch
     loop_control:
       loop_var: svc_name
 
@@ -50,6 +50,7 @@
       name: "{{ svc_name }}"
     when: "'opensearchDashboards' in group_names"
     loop:
+    - java-1.0.1
     - cerebro
     - opensearchDashboards
     - nodejs-1.0.0

diff --git a/ansible/roles/node-opensearch/files/etc/confd/templates/opensearch.sh/04.opensearch.yml.tmpl b/ansible/roles/node-opensearch/files/etc/confd/templates/opensearch.sh/04.opensearch.yml.tmpl
@@ -24,13 +24,9 @@ bootstrap.memory_lock: true
 
 cluster.name: {{ getv "/cluster/cluster_id" }}
 
-{{- if not (getvs "/env/es_additional_line*" | filter "cluster.routing.allocation.awareness.attributes:.*") }}
-cluster.routing.allocation.awareness.attributes: zone
-{{- end }}
-
 cluster.initial_master_nodes: [ ${discoveryHosts// /,} ]
 cluster.auto_shrink_voting_configuration: {{ getv "/env/cluster.auto_shrink_voting_configuration" "true" }}
-cluster.no_master_block: {{ getv "/env/cluster.no_master_block" "write" }}
+cluster.no_master_block: {{ getv "/env/discovery.zen.no_master_block" "write" }}
 
 discovery.seed_hosts: [ ${discoveryHosts// /,} ]
 

diff --git a/...roles/node-opensearch/files/etc/confd/templates/opensearch.sh/10.IKAnanlyzer.cfg.xml.tmpl b/...roles/node-opensearch/files/etc/confd/templates/opensearch.sh/10.IKAnanlyzer.cfg.xml.tmpl
@@ -3,7 +3,7 @@ flush > /opt/opensearch/current/plugins/opensearch-analisys-ik/config/IKAnalyzer
 <!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
 <properties>
   <entry key="ext_dict">custom/jieba.dic;extra_main.dic</entry>
-  <entry key="ext_stopwords">custom/stop_words.dic</entry>
+  <entry key="ext_stopwords">custom/extra_stopword.dic</entry>
   <entry key="remote_ext_dict">{{ getv "/env/remote_ext_dict" "" }}</entry>
   <entry key="remote_ext_stopwords">{{ getv "/env/remote_ext_stopwords" "" }}</entry>
 </properties>

diff --git a/ansible/roles/node-opensearch/files/opt/app/bin/node/opensearch.sh b/ansible/roles/node-opensearch/files/opt/app/bin/node/opensearch.sh
@@ -196,6 +196,12 @@ scale() {
   fi
 }
 
+checkMasterRemoved2() {
+  local res=$(curl -s -u 'admin:Change1Pwd' $MY_IP:9200/_cat/nodes | grep \* | cut -d' ' -f1)
+  test -n "$res"
+  test "$res" != "$@"
+}
+
 destroy() {
   # In case the user is trying to remove all ES nodes, when preScaleIn will never be called.
   if [ -n "$LEAVING_DATA_NODES" ]; then
@@ -206,15 +212,22 @@ destroy() {
   # https://www.elastic.co/guide/en/elasticsearch/reference/7.5/modules-discovery-adding-removing-nodes.html#modules-discovery-removing-nodes
   local masterNodesToLeave; masterNodesToLeave="$(getMasterNodesToExclude)"
   if [[ " $masterNodesToLeave " == *" $MY_IP "* ]]; then
-    local runningNodes
-    runningNodes="$(curl -s -m5 "$MY_IP:9200/_cat/nodes?h=i,id&full_id=true -u ${MY_ADMIN_USER}:${MY_ADMIN_PASSWORD}" | awk '{print $1"/"$2}')"
-    local node; for node in $masterNodesToLeave; do
-      if [ "$node" = "$MY_IP" ]; then break; fi
-      retry 120 1 0 checkPortClosed $node
-      local nodeId; nodeId=$(echo "$runningNodes" | awk -F/ '$1=="'$node'" {print $2}')
-      test -n "$nodeId"
-      retry 60 1 0 checkMasterRemoved $nodeId
+    local tmplist=($masterNodesToLeave)
+    local cnt=${#tmplist[@]}
+    local prenode=""
+    for((i=0;i<$cnt;i++)); do
+      if [ ${tmplist[i]} = "$MY_IP" ]; then
+        if [ $i -gt 0 ]; then
+          prenode=${tmplist[$((i-1))]}
+        fi
+        break
+      fi
     done
+
+    if [ -n "$prenode" ]; then
+      retry 600 2 0 checkClusterHealthy
+      retry 60 1 0 checkMasterRemoved2 $prenode
+    fi
     execute stop
   fi
 

diff --git a/ansible/roles/node-opensearch/tasks/main.yml b/ansible/roles/node-opensearch/tasks/main.yml
@@ -45,4 +45,4 @@
     force: yes
     owner: opensearch
     group: svc
-    state: link
+    state: link
diff --git a/ansible/roles/opensearch/files/opt/app/conf/opensearch/dicts.policy b/ansible/roles/opensearch/files/opt/app/conf/opensearch/dicts.policy
@@ -1,18 +1,7 @@
 grant {
   permission java.io.FilePermission "/data/opensearch/analysis/*", "read";
-  permission java.net.SocketPermission "*:*", "connect,resolve";
-  permission java.lang.RuntimePermission "ClassInPackage.sun.misc";
-  permission java.lang.RuntimePermission "DeclaredMembers";
-  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
-
-  permission java.lang.RuntimePermission "accessClassInPackage.sun.java2d.cmm.kcms";
-  permission java.lang.RuntimePermission "accessDeclaredMembers";
-  permission java.lang.RuntimePermission "getClassLoader";
-  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
-  permission java.security.SecurityPermission "createAccessControlContext";
-  permission java.security.SecurityPermission "insertProvider";
-  permission java.security.SecurityPermission "putProviderProperty.BC";
-
+  permission java.net.SocketPermission "*", "connect,resolve";
+  permission java.net.SocketPermission "localhost:0", "listen,resolve";
   permission java.lang.RuntimePermission "accessClassInPackage.sun.security.krb5";
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.RuntimePermission "getClassLoader";
@@ -22,18 +11,22 @@ grant {
   permission java.lang.RuntimePermission "setContextClassLoader";
   permission java.lang.RuntimePermission "shutdownHooks";
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
-  permission java.net.SocketPermission "*", "connect,resolve";
-  permission java.net.SocketPermission "localhost:0", "listen,resolve";
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.java2d.cmm.kcms";
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+  permission java.util.PropertyPermission "*", "read,write";
+  permission java.util.PropertyPermission "opensearch.allow_insecure_settings", "read,write";
   permission java.security.SecurityPermission "insertProvider.SaslPlainServer";
   permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
-  permission java.util.PropertyPermission "*", "read,write";
+  permission java.security.SecurityPermission "createAccessControlContext";
+  permission java.security.SecurityPermission "insertProvider";
+  permission java.security.SecurityPermission "putProviderProperty.BC";
   permission javax.security.auth.AuthPermission "doAs";
   permission javax.security.auth.AuthPermission "getSubject";
   permission javax.security.auth.AuthPermission "modifyPrincipals";
   permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
   permission javax.security.auth.AuthPermission "modifyPublicCredentials";
-  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosTicket * "*"", "read";
-  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * "*"", "read";
-  permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * "*"", "read";
+  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosTicket * \"*\"", "read";
+  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read";
+  permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
   permission javax.security.auth.kerberos.ServicePermission "*", "initiate";
 };
diff --git a/ansible/roles/opensearch/tasks/install-plugin2.yml b/ansible/roles/opensearch/tasks/install-plugin2.yml
@@ -0,0 +1,16 @@
+---
+- name: Install OS plugins - {{ opts.name }}
+  shell: |
+    extra_url1="https://github.com/aparo/opensearch-analysis-ik/releases/download/{{ opensearch_version }}/opensearch-analisys-ik-{{ opensearch_version }}.zip"
+    extra_url2="https://github.com/aparo/opensearch-prometheus-exporter/releases/download/{{ opensearch_version }}/prometheus-exporter-{{ opensearch_version }}.zip"
+    realname="{{ opts.name }}"
+    res=$(/opt/opensearch/current/bin/opensearch-plugin list | grep "$realname" | wc -l)
+    if [ "$res" -eq 1 ]; then exit 0; fi
+    if [ "$realname" = "opensearch-analisys-ik" ]; then
+      realname="$extra_url1"
+    elif [ "$realname" = "prometheus-exporter" ]; then
+      realname="$extra_url2"
+    fi
+    /opt/opensearch/current/bin/opensearch-plugin install -v --batch "$realname"
+  register: installed
+  ignore_errors: True
diff --git a/ansible/roles/opensearch/tasks/main.yml b/ansible/roles/opensearch/tasks/main.yml
@@ -13,7 +13,7 @@
     version: "{{ opensearch_version }}"
     parentRole: opensearch
   loop:
-    - pkgUrl: "https://artifacts.opensearch.org/releases/bundle/opensearch/{{ version }}/opensearch-{{ package_version }}-linux-x64.tar.gz"
+    - pkgUrl: "https://artifacts.opensearch.org/releases/bundle/opensearch/{{ opensearch_version }}/opensearch-{{ opensearch_version }}-linux-x64.tar.gz"
   loop_control:
     loop_var: opts
 
@@ -48,74 +48,95 @@
     masked: yes
     state: stopped
 
-#- name: install plugins
-#  include_tasks: install-plugin.yml
-#  loop:
-#    - name: analysis-icu
-#    - name: analysis-kuromoji
-#    - name: analysis-nori
-#    - name: analysis-phonetic
-#    - name: analysis-smartcn
-#    - name: analysis-stempel
-#    - name: analysis-ukrainian
-#    - name: mapper-annotated-text
-#    - name: mapper-murmur3
-#    - name: mapper-size
-    #- name: "https://github.com/aparo/opensearch-analysis-ik/releases/download/1.2.4/opensearch-analisys-ik-1.2.4.zip"
-    #  Below needed Manual installation, because it needs to be confirmed（yes）
-    #- name: repository-hdfs
-    #- name: repository-s3
-    #- name: ingest-attachment
-#  loop_control:
-#    loop_var: opts
-
-#- name: prepare dict directory
-#  file:
-#    path: /opt/opensearch/current/plugins/opensearch-analisys-ik/config/custom
-#    owner: root
-#    group: svc
-#    state: directory
-#
-#- name: check if jieba dic exists
-#  stat:
-#    path: "{{ role_path }}/files/tmp/jieba.dic"
-#  register: jieba_dic
-#  delegate_to: localhost
-#
-#- name: download jieba dict
-#  get_url:
-#    url: "https://github.com/QingCloudAppcenter/elk-archived/raw/master/docker-image/standard/elasticsearch/dicts/jieba.dic"
-#    dest: "{{ role_path }}/files/tmp/jieba.dic"
-#  delegate_to: localhost
-#  when: jieba_dic.stat.exists == False
-#
-#- name: copy jieba dict
-#  copy:
-#    src: "{{ role_path }}/files/tmp/jieba.dic"
-#    dest: "/opt/opensearch/current/plugins/analysis-ik/config/custom/jieba.dic"
-#    owner: root
-#    group: svc
-#    mode: u=rw,go=r
-#
-#- name: check if package file exists
-#  stat:
-#    path: "{{ role_path }}/files/tmp/extra_stopword.dic"
-#  register: stopword_dic
-#  delegate_to: localhost
-#
-#- name: download dict for IKAnalyzer
-#  get_url:
-#    url: "https://github.com/QingCloudAppcenter/elk-archived/raw/master/docker-image/standard/elasticsearch/dicts/stop_words.dic"
-#    dest: "{{ role_path }}/files/tmp/extra_stopword.dic"
-#  delegate_to: localhost
-#  when: stopword_dic.stat.exists == False
-#
-#- name: copy extra stopword dict
-#  copy:
-#    src: "{{ role_path }}/files/tmp/extra_stopword.dic"
-#    dest: "/opt/opensearch/current/plugins/opensearch-analisys-ik/config/custom/extra_stopword.dic"
-#    owner: root
-#    group: svc
+- name: install plugins
+  include_tasks: install-plugin2.yml
+  loop:
+    - name: analysis-icu
+    - name: analysis-kuromoji
+    - name: analysis-nori
+    - name: analysis-phonetic
+    - name: analysis-smartcn
+    - name: analysis-stempel
+    - name: analysis-ukrainian
+    - name: mapper-annotated-text
+    - name: mapper-murmur3
+    - name: mapper-size
+    - name: opensearch-analisys-ik
+    - name: repository-hdfs
+    - name: repository-s3
+    - name: ingest-attachment
+    - name: prometheus-exporter
+  loop_control:
+    loop_var: opts
+
+- name: prepare dict directory
+  file:
+    path: /opt/opensearch/current/plugins/opensearch-analisys-ik/config/custom
+    owner: opensearch
+    group: svc
+    state: directory
+
+- name: check if jieba dic exists
+  stat:
+    path: "{{ role_path }}/files/tmp/jieba.dic"
+  register: jieba_dic
+  delegate_to: localhost
+
+- name: download jieba dict
+  get_url:
+    url: "https://github.com/QingCloudAppcenter/elk-archived/raw/master/docker-image/standard/elasticsearch/dicts/jieba.dic"
+    dest: "{{ role_path }}/files/tmp/jieba.dic"
+  delegate_to: localhost
+  when: jieba_dic.stat.exists == False
+
+- name: copy jieba dict
+  copy:
+    src: "{{ role_path }}/files/tmp/jieba.dic"
+    dest: "/opt/opensearch/current/plugins/opensearch-analisys-ik/config/custom/jieba.dic"
+    owner: opensearch
+    group: svc
+    mode: u=rw,go=r
+
+- name: check if package file exists
+  stat:
+    path: "{{ role_path }}/files/tmp/extra_stopword.dic"
+  register: stopword_dic
+  delegate_to: localhost
+
+- name: download dict for IKAnalyzer
+  get_url:
+    url: "https://github.com/QingCloudAppcenter/elk-archived/raw/master/docker-image/standard/elasticsearch/dicts/stop_words.dic"
+    dest: "{{ role_path }}/files/tmp/extra_stopword.dic"
+  delegate_to: localhost
+  when: stopword_dic.stat.exists == False
+
+- name: copy extra stopword dict
+  copy:
+    src: "{{ role_path }}/files/tmp/extra_stopword.dic"
+    dest: "/opt/opensearch/current/plugins/opensearch-analisys-ik/config/custom/extra_stopword.dic"
+    owner: opensearch
+    group: svc
+
+- name: check if mid_file exists
+  stat:
+    path: "{{ role_path }}/files/tmp/xxx"
+  register: mid_file
+  delegate_to: localhost
+
+- name: download mid_file
+  get_url:
+    url: "https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.10.2/elasticsearch-analysis-ik-7.10.2.zip"
+    dest: "{{ role_path }}/files/tmp/mid_file.zip"
+  delegate_to: localhost
+  when: mid_file.stat.exists == False
+
+- name: unzip mid_file.zip
+  unarchive:
+    src: "{{ role_path }}/files/tmp/mid_file.zip"
+    dest: "/opt/opensearch/current/plugins/opensearch-analisys-ik"
+    include: "config/*"
+    owner: opensearch
+    group: svc
 
 
 
diff --git a/ansible/roles/opensearchDashboards/tasks/main.yml b/ansible/roles/opensearchDashboards/tasks/main.yml
@@ -13,7 +13,7 @@
     version: "{{ opensearch_dashboards_version }}"
     parentRole: dashboards
   loop:
-    - pkgUrl: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/{{ version }}/opensearch-dashboards-{{ opensearch_dashboards_package_version }}-linux-x64.tar.gz"
+    - pkgUrl: "https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/{{ opensearch_dashboards_version }}/opensearch-dashboards-{{ opensearch_dashboards_package_version }}-linux-x64.tar.gz"
   loop_control:
     loop_var: opts
 
@@ -43,7 +43,7 @@
 
 - name: mask opensearch dashboards service
   systemd:
-    name: opensearchDashboards
+    name: dashboards
     enabled: no
     masked: yes
     state: stopped