November 2019 - tor-commits - lists.torproject.org

[dip/master] Upgrade to 12.3.2
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit 2d965106933391e43b6446b81ad7b3844715a537 Author: Bastian Blank <waldi(a)debian.org> Date: Mon Sep 30 21:43:17 2019 +0200 Upgrade to 12.3.2 --- playbooks/group_vars/all.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/group_vars/all.yml b/playbooks/group_vars/all.yml index 1f4cf8f..775faf1 100644 --- a/playbooks/group_vars/all.yml +++ b/playbooks/group_vars/all.yml @@ -1,10 +1,10 @@ --- gitlab_repo: https://salsa.debian.org/salsa/gitlab.git -gitlab_version: v12.3.1-salsa-1 +gitlab_version: v12.3.2-salsa-1 gitlab_pages_version: v1.9.0 gitlab_shell_version: v10.0.0 gitlab_workhorse_version: v8.10.0 -gitaly_version: v1.65.0 +gitaly_version: v1.65.1 gitlab_runner_version: v12.2.0 docker_machine_version: v0.16.1 docker_registry_version: v2.7.1

1 0

[dip/master] Upgrade to 12.4.1
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit f794ac5fa657bed7041f724ee1b115faa6660b68 Author: Bastian Blank <waldi(a)debian.org> Date: Thu Oct 31 21:05:42 2019 +0100 Upgrade to 12.4.1 --- playbooks/group_vars/all.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/group_vars/all.yml b/playbooks/group_vars/all.yml index a689bec..4d587d4 100644 --- a/playbooks/group_vars/all.yml +++ b/playbooks/group_vars/all.yml @@ -1,11 +1,11 @@ --- gitlab_repo: https://salsa.debian.org/salsa/gitlab.git -gitlab_version: v12.4.0-salsa-1 +gitlab_version: v12.4.1-salsa-1 gitlab_pages_version: v1.11.0 gitlab_shell_version: v10.2.0 gitlab_workhorse_version: v8.14.0 gitaly_version: v1.67.0 -gitlab_runner_version: v12.4.0 +gitlab_runner_version: v12.4.1 docker_machine_version: v0.16.1 docker_registry_version: v2.7.1 minio_version: RELEASE.2018-07-23T18-34-49Z

1 0

[dip/master] Upgrade to 12.4.2
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit 691def12fc2c9b31fefe462161e840fa457a477f Author: Bastian Blank <waldi(a)debian.org> Date: Tue Nov 5 19:50:04 2019 +0100 Upgrade to 12.4.2 --- playbooks/group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/group_vars/all.yml b/playbooks/group_vars/all.yml index 4d587d4..928a0ca 100644 --- a/playbooks/group_vars/all.yml +++ b/playbooks/group_vars/all.yml @@ -1,6 +1,6 @@ --- gitlab_repo: https://salsa.debian.org/salsa/gitlab.git -gitlab_version: v12.4.1-salsa-1 +gitlab_version: v12.4.2-salsa-1 gitlab_pages_version: v1.11.0 gitlab_shell_version: v10.2.0 gitlab_workhorse_version: v8.14.0

1 0

[dip/master] Upgrade to 12.4.0
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit efce4711d417b0887cad61e917d276a524b6bcb2 Author: Bastian Blank <waldi(a)debian.org> Date: Wed Oct 23 16:55:05 2019 +0200 Upgrade to 12.4.0 --- playbooks/group_vars/all.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/playbooks/group_vars/all.yml b/playbooks/group_vars/all.yml index 775faf1..a689bec 100644 --- a/playbooks/group_vars/all.yml +++ b/playbooks/group_vars/all.yml @@ -1,11 +1,11 @@ --- gitlab_repo: https://salsa.debian.org/salsa/gitlab.git -gitlab_version: v12.3.2-salsa-1 -gitlab_pages_version: v1.9.0 -gitlab_shell_version: v10.0.0 -gitlab_workhorse_version: v8.10.0 -gitaly_version: v1.65.1 -gitlab_runner_version: v12.2.0 +gitlab_version: v12.4.0-salsa-1 +gitlab_pages_version: v1.11.0 +gitlab_shell_version: v10.2.0 +gitlab_workhorse_version: v8.14.0 +gitaly_version: v1.67.0 +gitlab_runner_version: v12.4.0 docker_machine_version: v0.16.1 docker_registry_version: v2.7.1 minio_version: RELEASE.2018-07-23T18-34-49Z

1 0

[dip/master] Upgrade to elasticsearch 7
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit fc2c101b6362a260b62b9fc7c942d0aee35602ee Author: Bastian Blank <waldi(a)debian.org> Date: Tue Nov 5 21:14:21 2019 +0100 Upgrade to elasticsearch 7 --- roles/system/logs/tasks/elasticsearch.yml | 2 +- roles/system/logs/templates/elasticsearch/elasticsearch.yml.j2 | 2 +- roles/system/logs/templates/kibana/kibana.yml.j2 | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/system/logs/tasks/elasticsearch.yml b/roles/system/logs/tasks/elasticsearch.yml index 8d68b34..087e1e0 100644 --- a/roles/system/logs/tasks/elasticsearch.yml +++ b/roles/system/logs/tasks/elasticsearch.yml @@ -45,7 +45,7 @@ - name: setup elasticsearch repo copy: content: | - deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main + deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main dest: /etc/apt/sources.list.d/elasticsearch.list - name: install elasticsearch diff --git a/roles/system/logs/templates/elasticsearch/elasticsearch.yml.j2 b/roles/system/logs/templates/elasticsearch/elasticsearch.yml.j2 index d072ec7..59a2f29 100644 --- a/roles/system/logs/templates/elasticsearch/elasticsearch.yml.j2 +++ b/roles/system/logs/templates/elasticsearch/elasticsearch.yml.j2 @@ -2,4 +2,4 @@ cluster.name: {{ external_domain_logs }} path.data: /srv/elasticsearch path.logs: /var/log/elasticsearch bootstrap.memory_lock: false -network.host: [_site_, _local_] +network.host: _local_ diff --git a/roles/system/logs/templates/kibana/kibana.yml.j2 b/roles/system/logs/templates/kibana/kibana.yml.j2 index 0450228..e69de29 100644 --- a/roles/system/logs/templates/kibana/kibana.yml.j2 +++ b/roles/system/logs/templates/kibana/kibana.yml.j2 @@ -1 +0,0 @@ -elasticsearch.url: "http://{{ ansible_default_ipv4.address }}:9200"

1 0

[dip/master] Upgrade to filebeat 7.3.0
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit 3423f48ff237ee8d9d7a8b279f26773ee3af74b1 Author: Bastian Blank <waldi(a)debian.org> Date: Tue Nov 5 23:19:02 2019 +0100 Upgrade to filebeat 7.3.0 --- playbooks/group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/group_vars/all.yml b/playbooks/group_vars/all.yml index 928a0ca..e663c8a 100644 --- a/playbooks/group_vars/all.yml +++ b/playbooks/group_vars/all.yml @@ -9,4 +9,4 @@ gitlab_runner_version: v12.4.1 docker_machine_version: v0.16.1 docker_registry_version: v2.7.1 minio_version: RELEASE.2018-07-23T18-34-49Z -filebeat_version: v6.7.1 +filebeat_version: v7.3.0

1 0

[dip/master] Ingest using new elasticsearch pipeline
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit 025034060f9fa3b6081c88886739b2eee26b1b23 Author: Bastian Blank <waldi(a)debian.org> Date: Tue Nov 5 23:18:36 2019 +0100 Ingest using new elasticsearch pipeline --- roles/gitlab-workhorse/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/gitlab-workhorse/meta/main.yml b/roles/gitlab-workhorse/meta/main.yml index e9cd3a0..4ed5b20 100644 --- a/roles/gitlab-workhorse/meta/main.yml +++ b/roles/gitlab-workhorse/meta/main.yml @@ -32,4 +32,4 @@ dependencies: paths: - /srv/{{ external_domain }}/log/gitlab-workhorse.log fields: - source: gitlab-workhorse + source: gitlab-workhorse-2

1 0

[dip/master] Make gitlab-workhorse elasticsearch 7 ready
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit d87c2870d356e8a77c8b60c5a94d0f7b45eabfb3 Author: Bastian Blank <waldi(a)debian.org> Date: Tue Nov 5 20:22:20 2019 +0100 Make gitlab-workhorse elasticsearch 7 ready --- .../gitlab-workhorse-2-access-pipeline.json | 87 +++++ .../gitlab-workhorse-2-msg-pipeline.json | 5 + .../elasticsearch/gitlab-workhorse-2-pipeline.json | 29 ++ .../elasticsearch/gitlab-workhorse-2-template.json | 358 +++++++++++++++++++++ .../elasticsearch/gitlab-workhorse-pipeline.json | 65 ---- .../elasticsearch/gitlab-workhorse-template.json | 184 ----------- roles/system/logs/tasks/elasticsearch.yml | 10 +- 7 files changed, 487 insertions(+), 251 deletions(-) diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json new file mode 100644 index 0000000..1e5e41c --- /dev/null +++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json @@ -0,0 +1,87 @@ +{ + "description": "Inject gitlab-workhorse log", + "processors": [ + { + "rename": { + "field": "gitlab-workhorse.method", + "target_field": "http.request.method" + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event = ['duration': Math.round(ctx['gitlab-workhorse'].duration_ms * 1000)]" + } + }, + { + "rename": { + "field": "gitlab-workhorse.remote_ip", + "target_field": "source.ip" + } + }, + { + "rename": { + "field": "gitlab-workhorse.referrer", + "target_field": "http.request.referrer" + } + }, + { + "rename": { + "field": "gitlab-workhorse.status", + "target_field": "http.response.status_code" + } + }, + { + "rename": { + "field": "gitlab-workhorse.uri", + "target_field": "url.original" + } + }, + { + "rename": { + "field": "gitlab-workhorse.user_agent", + "target_field": "user_agent.original" + } + }, + { + "rename": { + "field": "gitlab-workhorse.written_bytes", + "target_field": "http.response.body.bytes" + } + }, + { + "urldecode": { + "field": "url.original" + } + }, + { + "user_agent": { + "field": "user_agent.original" + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo" + } + }, + { + "remove": { + "field": [ + "gitlab-workhorse.duration_ms", + "gitlab-workhorse.host", + "gitlab-workhorse.level", + "gitlab-workhorse.msg", + "gitlab-workhorse.proto", + "gitlab-workhorse.remoteAddr", + "gitlab-workhorse.remote_addr", + "gitlab-workhorse.system", + "gitlab-workhorse.time", + "gitlab-workhorse.user_agent", + "gitlab-workhorse.version" + ], + "ignore_missing": true + } + } + ] +} diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json new file mode 100644 index 0000000..3253b04 --- /dev/null +++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json @@ -0,0 +1,5 @@ +{ + "description": "Inject gitlab-workhorse log", + "processors": [ + ] +} diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json new file mode 100644 index 0000000..88235cc --- /dev/null +++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json @@ -0,0 +1,29 @@ +{ + "description": "Inject gitlab-workhorse log", + "processors": [ + { + "rename": { + "field": "json", + "target_field": "gitlab-workhorse" + } + }, + { + "date": { + "field": "gitlab-workhorse.time", + "formats": [ "ISO8601" ] + } + }, + { + "pipeline": { + "name": "gitlab-workhorse-2-access", + "if": "ctx['gitlab-workhorse'].msg == 'access'" + } + }, + { + "pipeline": { + "name": "gitlab-workhorse-2-msg", + "if": "ctx['gitlab-workhorse'].msg != 'access'" + } + } + ] +} diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json new file mode 100644 index 0000000..e3aff13 --- /dev/null +++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json @@ -0,0 +1,358 @@ +{ + "aliases": {}, + "index_patterns": [ + "gitlab-workhorse-2-*" + ], + "mappings": { + "_meta": { + "version": "2" + }, + "date_detection": false, + "dynamic_templates": [ + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "offset": { + "type": "long" + }, + "gitlab-workhorse": { + "properties": { + "correlation_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "float" + }, + "encoding": { + "ignore_above": 1024, + "type": "keyword" + }, + "file": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 16, + "type": "keyword" + }, + "msg": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "user": { + "properties": { + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "order": 1, + "settings": { + "index": { + "number_of_replicas": "0", + "number_of_routing_shards": "30", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } +} diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json deleted file mode 100644 index 3991e61..0000000 --- a/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "description": "Inject gitlab-workhorse log", - "processors": [ - { - "rename": { - "field": "json", - "target_field": "gitlab-workhorse" - } - }, - { - "rename": { - "field": "gitlab-workhorse.remoteIp", - "target_field": "gitlab-workhorse.remote_ip", - "ignore_missing": true - } - }, - { - "geoip": { - "field": "gitlab-workhorse.remote_ip", - "target_field": "gitlab-workhorse.geoip", - "ignore_missing": true - } - }, - { - "date": { - "field": "gitlab-workhorse.time", - "formats": [ "ISO8601" ] - } - }, - { - "rename": { - "field": "gitlab-workhorse.user_agent", - "target_field": "gitlab-workhorse.user_agent_original", - "ignore_missing": true - } - }, - { - "user_agent": { - "field": "gitlab-workhorse.user_agent_original", - "target_field": "gitlab-workhorse.user_agent", - "ignore_failure": true - } - }, - { - "rename": { - "field": "gitlab-workhorse.user_agent_original", - "target_field": "gitlab-workhorse.user_agent.original", - "ignore_missing": true - } - }, - { - "remove": { - "field": [ - "gitlab-workhorse.host", - "gitlab-workhorse.proto", - "gitlab-workhorse.remoteAddr", - "gitlab-workhorse.system", - "gitlab-workhorse.time", - "gitlab-workhorse.version" - ], - "ignore_missing": true - } - } - ] -} diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json deleted file mode 100644 index 29000e2..0000000 --- a/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "aliases": {}, - "index_patterns": [ - "gitlab-workhorse-*" - ], - "mappings": { - "doc": { - "_meta": { - "version": "1" - }, - "date_detection": false, - "dynamic_templates": [ - { - "fields": { - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "fields.*" - } - } - ], - "properties": { - "@timestamp": { - "type": "date" - }, - "beat": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "fields": { - "type": "object" - }, - "offset": { - "type": "long" - }, - "gitlab-workhorse": { - "properties": { - "correlation_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "type": "float" - }, - "encoding": { - "ignore_above": 1024, - "type": "keyword" - }, - "file": { - "ignore_above": 1024, - "type": "keyword" - }, - "geoip": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "level": { - "ignore_above": 16, - "type": "keyword" - }, - "method": { - "ignore_above": 16, - "type": "keyword" - }, - "msg": { - "ignore_above": 1024, - "type": "keyword" - }, - "referer": { - "ignore_above": 1024, - "type": "keyword" - }, - "remote_ip": { - "type": "ip" - }, - "remote_port": { - "type": "integer" - }, - "status": { - "type": "short" - }, - "uri": { - "ignore_above": 1024, - "type": "keyword" - }, - "user_agent": { - "properties": { - "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "index": false, - "norms": false, - "type": "text" - }, - "os": { - "ignore_above": 1024, - "type": "keyword" - }, - "os_major": { - "type": "long" - }, - "os_minor": { - "type": "long" - }, - "os_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "written": { - "type": "long" - } - } - } - } - } - }, - "order": 1, - "settings": { - "index": { - "number_of_replicas": "0", - "number_of_routing_shards": "30", - "number_of_shards": "1", - "refresh_interval": "5s" - } - } -} diff --git a/roles/system/logs/tasks/elasticsearch.yml b/roles/system/logs/tasks/elasticsearch.yml index cd712d3..8d68b34 100644 --- a/roles/system/logs/tasks/elasticsearch.yml +++ b/roles/system/logs/tasks/elasticsearch.yml @@ -80,7 +80,11 @@ dest: /srv/elasticsearch/{{ item }}-pipeline.json validate: "curl --fail --retry 20 --retry-connrefused -X PUT -d @%s -H 'Content-Type: application/json' http://localhost:9200/_ingest/pipeline/{{ item }}" loop: - - gitlab-workhorse + - gitlab-workhorse-2 + - gitlab-workhorse-2-access + - gitlab-workhorse-2-msg + tags: + - elasticsearch-data - name: setup elasticsearch index template copy: @@ -88,4 +92,6 @@ dest: /srv/elasticsearch/{{ item }}-template.json validate: "curl --fail --retry 20 --retry-connrefused -X PUT -d @%s -H 'Content-Type: application/json' http://localhost:9200/_template/{{ item }}" loop: - - gitlab-workhorse + - gitlab-workhorse-2 + tags: + - elasticsearch-data

1 0

[dip/master] Add upgrade pipeline to new output
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit 8ff807b7134ec009752cab5db319e5ff5123bc8a Author: Bastian Blank <waldi(a)debian.org> Date: Tue Nov 5 23:14:19 2019 +0100 Add upgrade pipeline to new output --- .../gitlab-workhorse-2-upgrade-pipeline.json | 110 +++++++++++++++++++++ roles/system/logs/tasks/elasticsearch.yml | 1 + 2 files changed, 111 insertions(+) diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-upgrade-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-upgrade-pipeline.json new file mode 100644 index 0000000..3268884 --- /dev/null +++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-upgrade-pipeline.json @@ -0,0 +1,110 @@ +{ + "description": "Inject gitlab-workhorse log", + "processors": [ + { + "remove": { + "field": "source", + "ignore_missing": true + } + }, + { + "rename": { + "field": "offset", + "target_field": "log.offset" + } + }, + { + "rename": { + "field": "gitlab-workhorse.geoip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "rename": { + "field": "gitlab-workhorse.method", + "target_field": "http.request.method", + "ignore_missing": true + } + }, + { + "rename": { + "field": "gitlab-workhorse.remote_ip", + "target_field": "source.ip", + "ignore_missing": true + } + }, + { + "rename": { + "field": "gitlab-workhorse.referrer", + "target_field": "http.request.referrer", + "ignore_missing": true + } + }, + { + "rename": { + "field": "gitlab-workhorse.status", + "target_field": "http.response.status_code", + "ignore_missing": true + } + }, + { + "rename": { + "field": "gitlab-workhorse.uri", + "target_field": "url.original", + "ignore_missing": true + } + }, + { + "rename": { + "field": "gitlab-workhorse.user_agent.original", + "target_field": "user_agent.original", + "ignore_missing": true + } + }, + { + "rename": { + "field": "gitlab-workhorse.written", + "target_field": "http.response.body.bytes", + "ignore_missing": true + } + }, + { + "rename": { + "field": "gitlab-workhorse.written_bytes", + "target_field": "http.response.body.bytes", + "ignore_missing": true + } + }, + { + "urldecode": { + "field": "url.original", + "ignore_missing": true + } + }, + { + "user_agent": { + "field": "user_agent.original", + "ignore_missing": true + } + }, + { + "remove": { + "field": [ + "gitlab-workhorse.duration_ms", + "gitlab-workhorse.host", + "gitlab-workhorse.level", + "gitlab-workhorse.msg", + "gitlab-workhorse.proto", + "gitlab-workhorse.remoteAddr", + "gitlab-workhorse.remote_addr", + "gitlab-workhorse.system", + "gitlab-workhorse.time", + "gitlab-workhorse.user_agent", + "gitlab-workhorse.version" + ], + "ignore_missing": true + } + } + ] +} diff --git a/roles/system/logs/tasks/elasticsearch.yml b/roles/system/logs/tasks/elasticsearch.yml index 087e1e0..1805c3a 100644 --- a/roles/system/logs/tasks/elasticsearch.yml +++ b/roles/system/logs/tasks/elasticsearch.yml @@ -83,6 +83,7 @@ - gitlab-workhorse-2 - gitlab-workhorse-2-access - gitlab-workhorse-2-msg + - gitlab-workhorse-2-upgrade tags: - elasticsearch-data

1 0

[dip/master] Modify submodules
by hiro＠torproject.org 21 Nov '19

21 Nov '19

commit 416484527fad1a3e651eba265e3a66ab586a82ee Author: hiro <hiro(a)torproject.org> Date: Tue Oct 1 17:53:21 2019 +0200 Modify submodules --- .gitmodules | 5 +---- inventories/prod/group_vars/all.yml | 16 +++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.gitmodules b/.gitmodules index 30e2d41..9788653 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "inventories/prod/secrets"] path = inventories/prod/secrets - url = git@salsa.debian.org:salsa/salsa-ansible-secrets-prod.git -[submodule "inventories/test/secrets"] - path = inventories/test/secrets - url = git@salsa.debian.org:salsa/salsa-ansible-secrets-test.git + url = https://git.torproject.org/admin/services/gitlab/dip-configs.git diff --git a/inventories/prod/group_vars/all.yml b/inventories/prod/group_vars/all.yml index 91d48b7..8fa8e97 100644 --- a/inventories/prod/group_vars/all.yml +++ b/inventories/prod/group_vars/all.yml @@ -1,8 +1,10 @@ --- -external_domain: salsa.debian.org -external_domain_logs: salsa-logs.debian.net -external_domain_pages: pages.debian.net -external_domain_registry: registry.salsa.debian.org -ldapsync_filter: (supplementaryGid=Debian) -gitlab_storage_artifacts: debian-salsa-prod-gitlab-artifacts -gitlab_storage_lfs: debian-salsa-prod-gitlab-lfs +external_domain: dip.torproject.org +external_domain_pages: pages.dip.torproject.org +external_domain_registry: registry.dip.torproject.org +external_domain_logs: logs.dip.torproject.org +ldapsync_filter: (supplementaryGid=TorProject) +gitlab_storage_artifacts: torproject-dip-prod-gitlab-artifacts +gitlab_storage_lfs: torproject-dip-prod-gitlab-lfs +ldap_org: torproject +ldap_host: db.torproject.orgs

1 0