[tor-commits] [dip/master] Make gitlab-workhorse elasticsearch 7 ready
hiro at torproject.org
hiro at torproject.org
Thu Nov 21 10:02:52 UTC 2019
commit d87c2870d356e8a77c8b60c5a94d0f7b45eabfb3
Author: Bastian Blank <waldi at debian.org>
Date: Tue Nov 5 20:22:20 2019 +0100
Make gitlab-workhorse elasticsearch 7 ready
---
.../gitlab-workhorse-2-access-pipeline.json | 87 +++++
.../gitlab-workhorse-2-msg-pipeline.json | 5 +
.../elasticsearch/gitlab-workhorse-2-pipeline.json | 29 ++
.../elasticsearch/gitlab-workhorse-2-template.json | 358 +++++++++++++++++++++
.../elasticsearch/gitlab-workhorse-pipeline.json | 65 ----
.../elasticsearch/gitlab-workhorse-template.json | 184 -----------
roles/system/logs/tasks/elasticsearch.yml | 10 +-
7 files changed, 487 insertions(+), 251 deletions(-)
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json
new file mode 100644
index 0000000..1e5e41c
--- /dev/null
+++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json
@@ -0,0 +1,87 @@
+{
+ "description": "Inject gitlab-workhorse log",
+ "processors": [
+ {
+ "rename": {
+ "field": "gitlab-workhorse.method",
+ "target_field": "http.request.method"
+ }
+ },
+ {
+ "script": {
+ "lang": "painless",
+ "source": "ctx.event = ['duration': Math.round(ctx['gitlab-workhorse'].duration_ms * 1000)]"
+ }
+ },
+ {
+ "rename": {
+ "field": "gitlab-workhorse.remote_ip",
+ "target_field": "source.ip"
+ }
+ },
+ {
+ "rename": {
+ "field": "gitlab-workhorse.referrer",
+ "target_field": "http.request.referrer"
+ }
+ },
+ {
+ "rename": {
+ "field": "gitlab-workhorse.status",
+ "target_field": "http.response.status_code"
+ }
+ },
+ {
+ "rename": {
+ "field": "gitlab-workhorse.uri",
+ "target_field": "url.original"
+ }
+ },
+ {
+ "rename": {
+ "field": "gitlab-workhorse.user_agent",
+ "target_field": "user_agent.original"
+ }
+ },
+ {
+ "rename": {
+ "field": "gitlab-workhorse.written_bytes",
+ "target_field": "http.response.body.bytes"
+ }
+ },
+ {
+ "urldecode": {
+ "field": "url.original"
+ }
+ },
+ {
+ "user_agent": {
+ "field": "user_agent.original"
+ }
+ },
+ {
+ "geoip": {
+ "field": "source.ip",
+ "target_field": "source.geo"
+ }
+ },
+ {
+ "remove": {
+ "field": [
+ "gitlab-workhorse.duration_ms",
+ "gitlab-workhorse.host",
+ "gitlab-workhorse.level",
+ "gitlab-workhorse.msg",
+ "gitlab-workhorse.proto",
+ "gitlab-workhorse.remoteAddr",
+ "gitlab-workhorse.remote_addr",
+ "gitlab-workhorse.system",
+ "gitlab-workhorse.time",
+ "gitlab-workhorse.user_agent",
+ "gitlab-workhorse.version"
+ ],
+ "ignore_missing": true
+ }
+ }
+ ]
+}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json
new file mode 100644
index 0000000..3253b04
--- /dev/null
+++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json
@@ -0,0 +1,5 @@
+{
+ "description": "Inject gitlab-workhorse log",
+ "processors": [
+ ]
+}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json
new file mode 100644
index 0000000..88235cc
--- /dev/null
+++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json
@@ -0,0 +1,29 @@
+{
+ "description": "Inject gitlab-workhorse log",
+ "processors": [
+ {
+ "rename": {
+ "field": "json",
+ "target_field": "gitlab-workhorse"
+ }
+ },
+ {
+ "date": {
+ "field": "gitlab-workhorse.time",
+ "formats": [ "ISO8601" ]
+ }
+ },
+ {
+ "pipeline": {
+ "name": "gitlab-workhorse-2-access",
+ "if": "ctx['gitlab-workhorse'].msg == 'access'"
+ }
+ },
+ {
+ "pipeline": {
+ "name": "gitlab-workhorse-2-msg",
+ "if": "ctx['gitlab-workhorse'].msg != 'access'"
+ }
+ }
+ ]
+}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json
new file mode 100644
index 0000000..e3aff13
--- /dev/null
+++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json
@@ -0,0 +1,358 @@
+{
+ "aliases": {},
+ "index_patterns": [
+ "gitlab-workhorse-2-*"
+ ],
+ "mappings": {
+ "_meta": {
+ "version": "2"
+ },
+ "date_detection": false,
+ "dynamic_templates": [
+ {
+ "fields": {
+ "mapping": {
+ "type": "keyword"
+ },
+ "match_mapping_type": "string",
+ "path_match": "fields.*"
+ }
+ }
+ ],
+ "properties": {
+ "@timestamp": {
+ "type": "date"
+ },
+ "beat": {
+ "properties": {
+ "hostname": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "timezone": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "version": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "fields": {
+ "type": "object"
+ },
+ "offset": {
+ "type": "long"
+ },
+ "gitlab-workhorse": {
+ "properties": {
+ "correlation_id": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "duration": {
+ "type": "float"
+ },
+ "encoding": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "file": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "level": {
+ "ignore_above": 16,
+ "type": "keyword"
+ },
+ "msg": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "http": {
+ "properties": {
+ "request": {
+ "properties": {
+ "body": {
+ "properties": {
+ "bytes": {
+ "type": "long"
+ },
+ "content": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "bytes": {
+ "type": "long"
+ },
+ "method": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "referrer": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "response": {
+ "properties": {
+ "body": {
+ "properties": {
+ "bytes": {
+ "type": "long"
+ },
+ "content": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "bytes": {
+ "type": "long"
+ },
+ "status_code": {
+ "type": "long"
+ }
+ }
+ },
+ "version": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "source": {
+ "properties": {
+ "address": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "as": {
+ "properties": {
+ "number": {
+ "type": "long"
+ },
+ "organization": {
+ "properties": {
+ "name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "bytes": {
+ "type": "long"
+ },
+ "domain": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "geo": {
+ "properties": {
+ "city_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "continent_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "country_iso_code": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "country_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "location": {
+ "type": "geo_point"
+ },
+ "name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "region_iso_code": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "region_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "ip": {
+ "type": "ip"
+ },
+ "mac": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "packets": {
+ "type": "long"
+ },
+ "port": {
+ "type": "long"
+ },
+ "user": {
+ "properties": {
+ "email": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "full_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "group": {
+ "properties": {
+ "id": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "hash": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "id": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "url": {
+ "properties": {
+ "domain": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "fragment": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "full": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "original": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "password": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "path": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "port": {
+ "type": "long"
+ },
+ "query": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "scheme": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "username": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "user_agent": {
+ "properties": {
+ "device": {
+ "properties": {
+ "name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "original": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "os": {
+ "properties": {
+ "family": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "full": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "full_name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "kernel": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "name": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "platform": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "version": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ },
+ "version": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ }
+ }
+ }
+ }
+ },
+ "order": 1,
+ "settings": {
+ "index": {
+ "number_of_replicas": "0",
+ "number_of_routing_shards": "30",
+ "number_of_shards": "1",
+ "refresh_interval": "5s"
+ }
+ }
+}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json
deleted file mode 100644
index 3991e61..0000000
--- a/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
- "description": "Inject gitlab-workhorse log",
- "processors": [
- {
- "rename": {
- "field": "json",
- "target_field": "gitlab-workhorse"
- }
- },
- {
- "rename": {
- "field": "gitlab-workhorse.remoteIp",
- "target_field": "gitlab-workhorse.remote_ip",
- "ignore_missing": true
- }
- },
- {
- "geoip": {
- "field": "gitlab-workhorse.remote_ip",
- "target_field": "gitlab-workhorse.geoip",
- "ignore_missing": true
- }
- },
- {
- "date": {
- "field": "gitlab-workhorse.time",
- "formats": [ "ISO8601" ]
- }
- },
- {
- "rename": {
- "field": "gitlab-workhorse.user_agent",
- "target_field": "gitlab-workhorse.user_agent_original",
- "ignore_missing": true
- }
- },
- {
- "user_agent": {
- "field": "gitlab-workhorse.user_agent_original",
- "target_field": "gitlab-workhorse.user_agent",
- "ignore_failure": true
- }
- },
- {
- "rename": {
- "field": "gitlab-workhorse.user_agent_original",
- "target_field": "gitlab-workhorse.user_agent.original",
- "ignore_missing": true
- }
- },
- {
- "remove": {
- "field": [
- "gitlab-workhorse.host",
- "gitlab-workhorse.proto",
- "gitlab-workhorse.remoteAddr",
- "gitlab-workhorse.system",
- "gitlab-workhorse.time",
- "gitlab-workhorse.version"
- ],
- "ignore_missing": true
- }
- }
- ]
-}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json
deleted file mode 100644
index 29000e2..0000000
--- a/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json
+++ /dev/null
@@ -1,184 +0,0 @@
-{
- "aliases": {},
- "index_patterns": [
- "gitlab-workhorse-*"
- ],
- "mappings": {
- "doc": {
- "_meta": {
- "version": "1"
- },
- "date_detection": false,
- "dynamic_templates": [
- {
- "fields": {
- "mapping": {
- "type": "keyword"
- },
- "match_mapping_type": "string",
- "path_match": "fields.*"
- }
- }
- ],
- "properties": {
- "@timestamp": {
- "type": "date"
- },
- "beat": {
- "properties": {
- "hostname": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "timezone": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "version": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "fields": {
- "type": "object"
- },
- "offset": {
- "type": "long"
- },
- "gitlab-workhorse": {
- "properties": {
- "correlation_id": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "duration": {
- "type": "float"
- },
- "encoding": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "file": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "geoip": {
- "properties": {
- "city_name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "continent_name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "country_iso_code": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "location": {
- "type": "geo_point"
- },
- "region_iso_code": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "region_name": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "level": {
- "ignore_above": 16,
- "type": "keyword"
- },
- "method": {
- "ignore_above": 16,
- "type": "keyword"
- },
- "msg": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "referer": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "remote_ip": {
- "type": "ip"
- },
- "remote_port": {
- "type": "integer"
- },
- "status": {
- "type": "short"
- },
- "uri": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "user_agent": {
- "properties": {
- "device": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "major": {
- "type": "long"
- },
- "minor": {
- "type": "long"
- },
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "original": {
- "index": false,
- "norms": false,
- "type": "text"
- },
- "os": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "os_major": {
- "type": "long"
- },
- "os_minor": {
- "type": "long"
- },
- "os_name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
- "patch": {
- "ignore_above": 1024,
- "type": "keyword"
- }
- }
- },
- "written": {
- "type": "long"
- }
- }
- }
- }
- }
- },
- "order": 1,
- "settings": {
- "index": {
- "number_of_replicas": "0",
- "number_of_routing_shards": "30",
- "number_of_shards": "1",
- "refresh_interval": "5s"
- }
- }
-}
diff --git a/roles/system/logs/tasks/elasticsearch.yml b/roles/system/logs/tasks/elasticsearch.yml
index cd712d3..8d68b34 100644
--- a/roles/system/logs/tasks/elasticsearch.yml
+++ b/roles/system/logs/tasks/elasticsearch.yml
@@ -80,7 +80,11 @@
dest: /srv/elasticsearch/{{ item }}-pipeline.json
validate: "curl --fail --retry 20 --retry-connrefused -X PUT -d @%s -H 'Content-Type: application/json' http://localhost:9200/_ingest/pipeline/{{ item }}"
loop:
- - gitlab-workhorse
+ - gitlab-workhorse-2
+ - gitlab-workhorse-2-access
+ - gitlab-workhorse-2-msg
+ tags:
+ - elasticsearch-data
- name: setup elasticsearch index template
copy:
@@ -88,4 +92,6 @@
dest: /srv/elasticsearch/{{ item }}-template.json
validate: "curl --fail --retry 20 --retry-connrefused -X PUT -d @%s -H 'Content-Type: application/json' http://localhost:9200/_template/{{ item }}"
loop:
- - gitlab-workhorse
+ - gitlab-workhorse-2
+ tags:
+ - elasticsearch-data
More information about the tor-commits
mailing list