1.1.0 - v1.0.0

ansible/group_vars/all.yml

@@ -0,0 +1,5 @@
+appAgentVersion: v1.0.6
+opensearch_version: 1.1.0
+package_version: 1.1.0
+elk_version: 7.10.2
+logstash_version: 7.13.2

ansible/make.yml

@@ -0,0 +1,58 @@
+---
+- name: common setup
+  hosts: all
+  vars:
+    target_env: "{{ lookup('env', 'target') }}"
+  vars_files:
+  - /root/.qcacode
+  strategy: free
+  roles:
+  - app-agent-1.0.1
+  - appctl-1.0.9
+  - arping-1.0.0
+  - common
+  - role: caddy-1.0.6
+    vars:
+      caddy_plugins: upload
+  - update-apt-sources-1.0.0
+  - disable-apt-jobs-1.0.0
+  - disable-motd-1.0.0
+
+  tasks:
+  - include_role:
+      name: "{{ svc_name }}"
+    when: "'opensearch' in group_names"
+    loop:
+    - java-1.0.1
+    - opensearch
+    - jq-1.0.3
+    - node-opensearch
+    loop_control:
+      loop_var: svc_name
+
+  - include_role:
+      name: java-1.0.1
+    vars:
+      java_version: 8
+    when: ('logstash' in group_names) or ('' in group_names)
+
+  - include_role:
+      name: "{{ svc_name }}"
+    when: "'logstash' in group_names"
+    loop:
+    - logstash
+    - node-logstash
+    loop_control:
+      loop_var: svc_name
+
+  - include_role:
+      name: "{{ svc_name }}"
+    when: "'opensearchDashboards' in group_names"
+    loop:
+    - opensearchDashboards
+    - nodejs-1.0.0
+    - haproxy-1.0.3
+    - keepalived-1.0.1
+    - node-opensearchDashboards
+    loop_control:
+      loop_var: svc_name

ansible/requirements.yml

@@ -0,0 +1,13 @@
+- src: https://qingcloudappcenter.github.io/ansible-roles/app-agent-1.0.1.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/appctl-1.0.9.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/caddy-1.0.6.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/java-1.0.1.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/jq-1.0.3.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/nodejs-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/haproxy-1.0.3.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/keepalived-1.0.1.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.0.1.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/update-apt-sources-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/disable-apt-jobs-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/disable-motd-1.0.0.tar.gz

ansible/roles/common/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+- name: set common env
+  copy:
+    content: |
+      ELK_VERSION={{ opensearch_version }}
+
+    dest: /opt/app/bin/envs/all.env

ansible/roles/logstash/files/lib/systemd/system/logstash.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=logstash
+
+[Service]
+Type=simple
+User=logstash
+Group=svc
+# Load env vars from /etc/default/ and /etc/sysconfig/ if they exist.
+# Prefixing the path with '-' makes it try to load, but if the file doesn't
+# exist, it continues onward.
+EnvironmentFile=/opt/app/conf/logstash/.env
+ExecStart=/usr/share/logstash/bin/logstash
+Restart=always
+WorkingDirectory=/data/logstash
+Nice=19
+LimitNOFILE=1048576
+
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+NoNewPrivileges=true
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/logstash/files/opt/app/conf/logstash/.env

@@ -0,0 +1,2 @@
+LS_HOME=/usr/share/logstash
+LS_SETTINGS_DIR=/opt/app/conf/logstash

ansible/roles/logstash/files/opt/app/conf/logstash/log4j2.properties

@@ -0,0 +1,116 @@
+status = error
+name = LogstashPropertiesConfig
+
+appender.console.type = Console
+appender.console.name = plain_console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+
+appender.json_console.type = Console
+appender.json_console.name = json_console
+appender.json_console.layout.type = JSONLayout
+appender.json_console.layout.compact = true
+appender.json_console.layout.eventEol = true
+
+appender.rolling.type = RollingFile
+appender.rolling.name = plain_rolling
+appender.rolling.fileName = ${sys:ls.logs}/logstash-${sys:ls.log.format}.log
+appender.rolling.filePattern = ${sys:ls.logs}/logstash-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %-.10000m%n
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size = 2MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.action.type = Delete
+appender.rolling.strategy.action.basepath = ${sys:ls.logs}
+appender.rolling.strategy.action.condition.type = IfFileName
+appender.rolling.strategy.action.condition.glob = logstash-${sys:ls.logs.format}-*
+appender.rolling.strategy.action.condition.nested_condition.type = IfAny
+appender.rolling.strategy.action.condition.nested_condition.lastMod.type = IfLastModified
+appender.rolling.strategy.action.condition.nested_condition.lastMod.age = 7D
+appender.rolling.strategy.action.condition.nested_condition.fileSize.type = IfAccumulatedFileSize
+appender.rolling.strategy.action.condition.nested_condition.fileSize.exceeds = 200MB
+
+appender.json_rolling.type = RollingFile
+appender.json_rolling.name = json_rolling
+appender.json_rolling.fileName = ${sys:ls.logs}/logstash-${sys:ls.log.format}.log
+appender.json_rolling.filePattern = ${sys:ls.logs}/logstash-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log
+appender.json_rolling.layout.type = JSONLayout
+appender.json_rolling.layout.compact = true
+appender.json_rolling.layout.eventEol = true
+appender.json_rolling.policies.type = Policies
+appender.json_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.json_rolling.policies.time.interval = 1
+appender.json_rolling.policies.time.modulate = true
+appender.json_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.json_rolling.policies.size.size = 2MB
+appender.json_rolling.strategy.type = DefaultRolloverStrategy
+appender.json_rolling.strategy.action.type = Delete
+appender.json_rolling.strategy.action.basepath = ${sys:ls.logs}
+appender.json_rolling.strategy.action.condition.type = IfFileName
+appender.json_rolling.strategy.action.condition.glob = logstash-${sys:ls.logs.format}-*
+appender.json_rolling.strategy.action.condition.nested_condition.type = IfAny
+appender.json_rolling.strategy.action.condition.nested_condition.lastMod.type = IfLastModified
+appender.json_rolling.strategy.action.condition.nested_condition.lastMod.age = 7D
+appender.json_rolling.strategy.action.condition.nested_condition.fileSize.type = IfAccumulatedFileSize
+appender.json_rolling.strategy.action.condition.nested_condition.fileSize.exceeds = 200MB
+
+
+rootLogger.level = ${sys:ls.log.level}
+rootLogger.appenderRef.rolling.ref = ${sys:ls.log.format}_rolling
+
+# Slowlog
+
+appender.console_slowlog.type = Console
+appender.console_slowlog.name = plain_console_slowlog
+appender.console_slowlog.layout.type = PatternLayout
+appender.console_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+
+appender.json_console_slowlog.type = Console
+appender.json_console_slowlog.name = json_console_slowlog
+appender.json_console_slowlog.layout.type = JSONLayout
+appender.json_console_slowlog.layout.compact = true
+appender.json_console_slowlog.layout.eventEol = true
+
+appender.rolling_slowlog.type = RollingFile
+appender.rolling_slowlog.name = plain_rolling_slowlog
+appender.rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}.log
+appender.rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log
+appender.rolling_slowlog.policies.type = Policies
+appender.rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling_slowlog.policies.time.interval = 1
+appender.rolling_slowlog.policies.time.modulate = true
+appender.rolling_slowlog.layout.type = PatternLayout
+appender.rolling_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
+appender.rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling_slowlog.policies.size.size = 2MB
+appender.rolling_slowlog.strategy.type = DefaultRolloverStrategy
+appender.rolling_slowlog.strategy.max = 50
+
+appender.json_rolling_slowlog.type = RollingFile
+appender.json_rolling_slowlog.name = json_rolling_slowlog
+appender.json_rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}.log
+appender.json_rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log
+appender.json_rolling_slowlog.policies.type = Policies
+appender.json_rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy
+appender.json_rolling_slowlog.policies.time.interval = 1
+appender.json_rolling_slowlog.policies.time.modulate = true
+appender.json_rolling_slowlog.layout.type = JSONLayout
+appender.json_rolling_slowlog.layout.compact = true
+appender.json_rolling_slowlog.layout.eventEol = true
+appender.json_rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy
+appender.json_rolling_slowlog.policies.size.size = 2MB
+appender.json_rolling_slowlog.strategy.type = DefaultRolloverStrategy
+appender.json_rolling_slowlog.strategy.max = 50
+
+logger.slowlog.name = slowlog
+logger.slowlog.level = trace
+logger.slowlog.appenderRef.rolling_slowlog.ref = ${sys:ls.log.format}_rolling_slowlog
+logger.slowlog.additivity = false
+
+logger.licensereader.name = logstash.licensechecker.licensereader
+logger.licensereader.level = error

ansible/roles/logstash/files/opt/app/conf/logstash/template.json

@@ -0,0 +1,62 @@
+{
+  "index_patterns": "logstash-*",
+  "version": 70501,
+  "settings": {
+    "index": {
+      "refresh_interval": "5s",
+      "analysis": {
+        "analyzer": {
+          "by_smart": {
+            "type": "custom",
+            "tokenizer": "ik_smart"
+          },
+          "by_max_word": {
+            "type": "custom",
+            "tokenizer": "ik_max_word"
+          }
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [{
+      "message_field": {
+        "path_match": "message",
+        "match_mapping_type": "string",
+        "mapping": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "by_max_word",
+          "search_analyzer": "by_smart"
+        }
+      }
+    }, {
+      "string_fields": {
+        "match": "*",
+        "match_mapping_type": "string",
+        "mapping": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "by_max_word",
+          "search_analyzer": "by_smart",
+          "fields": {
+            "keyword": { "type": "keyword", "ignore_above": 256 }
+          }
+        }
+      }
+    }],
+    "properties": {
+      "@timestamp": { "type": "date" },
+      "@version": { "type": "keyword" },
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
+        }
+      }
+    }
+  }
+}