[tor-commits] [dip/master] Make gitlab-workhorse elasticsearch 7 ready

21 Nov 2019

commit d87c2870d356e8a77c8b60c5a94d0f7b45eabfb3
Author: Bastian Blank <waldi@debian.org>
Date:   Tue Nov 5 20:22:20 2019 +0100

    Make gitlab-workhorse elasticsearch 7 ready
---
 .../gitlab-workhorse-2-access-pipeline.json        |  87 +++++
 .../gitlab-workhorse-2-msg-pipeline.json           |   5 +
 .../elasticsearch/gitlab-workhorse-2-pipeline.json |  29 ++
 .../elasticsearch/gitlab-workhorse-2-template.json | 358 +++++++++++++++++++++
 .../elasticsearch/gitlab-workhorse-pipeline.json   |  65 ----
 .../elasticsearch/gitlab-workhorse-template.json   | 184 -----------
 roles/system/logs/tasks/elasticsearch.yml          |  10 +-
 7 files changed, 487 insertions(+), 251 deletions(-)

diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json
new file mode 100644
index 0000000..1e5e41c
--- /dev/null
+++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-access-pipeline.json
@@ -0,0 +1,87 @@
+{
+  "description": "Inject gitlab-workhorse log",
+  "processors": [
+    {
+      "rename": {
+        "field": "gitlab-workhorse.method",
+        "target_field": "http.request.method"
+      }
+    },
+    {
+      "script": {
+        "lang": "painless",
+        "source": "ctx.event = ['duration': Math.round(ctx['gitlab-workhorse'].duration_ms * 1000)]"
+      }
+    },
+    {
+      "rename": {
+        "field": "gitlab-workhorse.remote_ip",
+        "target_field": "source.ip"
+      }
+    },
+    {
+      "rename": {
+        "field": "gitlab-workhorse.referrer",
+        "target_field": "http.request.referrer"
+      }
+    },
+    {
+      "rename": {
+        "field": "gitlab-workhorse.status",
+        "target_field": "http.response.status_code"
+      }
+    },
+    {
+      "rename": {
+        "field": "gitlab-workhorse.uri",
+        "target_field": "url.original"
+      }
+    },
+    {
+      "rename": {
+        "field": "gitlab-workhorse.user_agent",
+        "target_field": "user_agent.original"
+      }
+    },
+    {
+      "rename": {
+        "field": "gitlab-workhorse.written_bytes",
+        "target_field": "http.response.body.bytes"
+      }
+    },
+    {
+      "urldecode": {
+        "field": "url.original"
+      }
+    },
+    {
+      "user_agent": {
+        "field": "user_agent.original"
+      }
+    },
+    {
+      "geoip": {
+        "field": "source.ip",
+        "target_field": "source.geo"
+      }
+    },
+    {
+      "remove": {
+        "field": [
+          "gitlab-workhorse.duration_ms",
+          "gitlab-workhorse.host",
+          "gitlab-workhorse.level",
+          "gitlab-workhorse.msg",
+          "gitlab-workhorse.proto",
+          "gitlab-workhorse.remoteAddr",
+          "gitlab-workhorse.remote_addr",
+          "gitlab-workhorse.system",
+          "gitlab-workhorse.time",
+          "gitlab-workhorse.user_agent",
+          "gitlab-workhorse.version"
+        ],
+        "ignore_missing": true
+      }
+    }
+  ]
+}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json
new file mode 100644
index 0000000..3253b04
--- /dev/null
+++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-msg-pipeline.json
@@ -0,0 +1,5 @@
+{
+  "description": "Inject gitlab-workhorse log",
+  "processors": [
+  ]
+}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json
new file mode 100644
index 0000000..88235cc
--- /dev/null
+++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-pipeline.json
@@ -0,0 +1,29 @@
+{
+  "description": "Inject gitlab-workhorse log",
+  "processors": [
+    {
+      "rename": {
+        "field": "json",
+        "target_field": "gitlab-workhorse"
+      }
+    },
+    {
+      "date": {
+        "field": "gitlab-workhorse.time",
+        "formats": [ "ISO8601" ]
+      }
+    },
+    {
+      "pipeline": {
+        "name": "gitlab-workhorse-2-access",
+        "if": "ctx['gitlab-workhorse'].msg == 'access'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "gitlab-workhorse-2-msg",
+        "if": "ctx['gitlab-workhorse'].msg != 'access'"
+      }
+    }
+  ]
+}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json
new file mode 100644
index 0000000..e3aff13
--- /dev/null
+++ b/roles/system/logs/files/elasticsearch/gitlab-workhorse-2-template.json
@@ -0,0 +1,358 @@
+{
+  "aliases": {},
+  "index_patterns": [
+    "gitlab-workhorse-2-*"
+  ],
+  "mappings": {
+    "_meta": {
+      "version": "2"
+    },
+    "date_detection": false,
+    "dynamic_templates": [
+      {
+        "fields": {
+          "mapping": {
+            "type": "keyword"
+          },
+          "match_mapping_type": "string",
+          "path_match": "fields.*"
+        }
+      }
+    ],
+    "properties": {
+      "@timestamp": {
+        "type": "date"
+      },
+      "beat": {
+        "properties": {
+          "hostname": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "name": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "timezone": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "version": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          }
+        }
+      },
+      "fields": {
+        "type": "object"
+      },
+      "offset": {
+        "type": "long"
+      },
+      "gitlab-workhorse": {
+        "properties": {
+          "correlation_id": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "duration": {
+            "type": "float"
+          },
+          "encoding": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "file": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "level": {
+            "ignore_above": 16,
+            "type": "keyword"
+          },
+          "msg": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          }
+        }
+      },
+      "http": {
+        "properties": {
+          "request": {
+            "properties": {
+              "body": {
+                "properties": {
+                  "bytes": {
+                    "type": "long"
+                  },
+                  "content": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              },
+              "bytes": {
+                "type": "long"
+              },
+              "method": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "referrer": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
+          "response": {
+            "properties": {
+              "body": {
+                "properties": {
+                  "bytes": {
+                    "type": "long"
+                  },
+                  "content": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              },
+              "bytes": {
+                "type": "long"
+              },
+              "status_code": {
+                "type": "long"
+              }
+            }
+          },
+          "version": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          }
+        }
+      },
+      "source": {
+        "properties": {
+          "address": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "as": {
+            "properties": {
+              "number": {
+                "type": "long"
+              },
+              "organization": {
+                "properties": {
+                  "name": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              }
+            }
+          },
+          "bytes": {
+            "type": "long"
+          },
+          "domain": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "geo": {
+            "properties": {
+              "city_name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "continent_name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "country_iso_code": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "country_name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "location": {
+                "type": "geo_point"
+              },
+              "name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "region_iso_code": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "region_name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
+          "ip": {
+            "type": "ip"
+          },
+          "mac": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "packets": {
+            "type": "long"
+          },
+          "port": {
+            "type": "long"
+          },
+          "user": {
+            "properties": {
+              "email": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "full_name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "group": {
+                "properties": {
+                  "id": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "name": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              },
+              "hash": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "id": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      },
+      "url": {
+        "properties": {
+          "domain": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "fragment": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "full": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "original": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "password": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "path": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "port": {
+            "type": "long"
+          },
+          "query": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "scheme": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "username": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          }
+        }
+      },
+      "user_agent": {
+        "properties": {
+          "device": {
+            "properties": {
+              "name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
+          "name": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "original": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "os": {
+            "properties": {
+              "family": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "full": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "full_name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "kernel": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "name": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "platform": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "version": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
+          "version": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  },
+  "order": 1,
+  "settings": {
+    "index": {
+      "number_of_replicas": "0",
+      "number_of_routing_shards": "30",
+      "number_of_shards": "1",
+      "refresh_interval": "5s"
+    }
+  }
+}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json
deleted file mode 100644
index 3991e61..0000000
--- a/roles/system/logs/files/elasticsearch/gitlab-workhorse-pipeline.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
-  "description": "Inject gitlab-workhorse log",
-  "processors": [
-    {
-      "rename": {
-        "field": "json",
-        "target_field": "gitlab-workhorse"
-      }
-    },
-    {
-      "rename": {
-        "field": "gitlab-workhorse.remoteIp",
-        "target_field": "gitlab-workhorse.remote_ip",
-        "ignore_missing": true
-      }
-    },
-    {
-      "geoip": {
-        "field": "gitlab-workhorse.remote_ip",
-        "target_field": "gitlab-workhorse.geoip",
-        "ignore_missing": true
-      }
-    },
-    {
-      "date": {
-        "field": "gitlab-workhorse.time",
-        "formats": [ "ISO8601" ]
-      }
-    },
-    {
-      "rename": {
-        "field": "gitlab-workhorse.user_agent",
-        "target_field": "gitlab-workhorse.user_agent_original",
-        "ignore_missing": true
-      }
-    },
-    {
-      "user_agent": {
-        "field": "gitlab-workhorse.user_agent_original",
-        "target_field": "gitlab-workhorse.user_agent",
-        "ignore_failure": true
-      }
-    },
-    {
-      "rename": {
-        "field": "gitlab-workhorse.user_agent_original",
-        "target_field": "gitlab-workhorse.user_agent.original",
-        "ignore_missing": true
-      }
-    },
-    {
-      "remove": {
-        "field": [
-          "gitlab-workhorse.host",
-          "gitlab-workhorse.proto",
-          "gitlab-workhorse.remoteAddr",
-          "gitlab-workhorse.system",
-          "gitlab-workhorse.time",
-          "gitlab-workhorse.version"
-        ],
-        "ignore_missing": true
-      }
-    }
-  ]
-}
diff --git a/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json b/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json
deleted file mode 100644
index 29000e2..0000000
--- a/roles/system/logs/files/elasticsearch/gitlab-workhorse-template.json
+++ /dev/null
@@ -1,184 +0,0 @@
-{
-  "aliases": {},
-  "index_patterns": [
-    "gitlab-workhorse-*"
-  ],
-  "mappings": {
-    "doc": {
-      "_meta": {
-        "version": "1"
-      },
-      "date_detection": false,
-      "dynamic_templates": [
-        {
-          "fields": {
-            "mapping": {
-              "type": "keyword"
-            },
-            "match_mapping_type": "string",
-            "path_match": "fields.*"
-          }
-        }
-      ],
-      "properties": {
-        "@timestamp": {
-          "type": "date"
-        },
-        "beat": {
-          "properties": {
-            "hostname": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "name": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "timezone": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "fields": {
-          "type": "object"
-        },
-        "offset": {
-          "type": "long"
-        },
-        "gitlab-workhorse": {
-          "properties": {
-            "correlation_id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "duration": {
-              "type": "float"
-            },
-            "encoding": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "file": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "geoip": {
-              "properties": {
-                "city_name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "continent_name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "country_iso_code": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "location": {
-                  "type": "geo_point"
-                },
-                "region_iso_code": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "region_name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "level": {
-              "ignore_above": 16,
-              "type": "keyword"
-            },
-            "method": {
-              "ignore_above": 16,
-              "type": "keyword"
-            },
-            "msg": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "referer": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "remote_ip": {
-              "type": "ip"
-            },
-            "remote_port": {
-              "type": "integer"
-            },
-            "status": {
-              "type": "short"
-            },
-            "uri": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "user_agent": {
-              "properties": {
-                "device": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "major": {
-                  "type": "long"
-                },
-                "minor": {
-                  "type": "long"
-                },
-                "name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "original": {
-                  "index": false,
-                  "norms": false,
-                  "type": "text"
-                },
-                "os": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "os_major": {
-                  "type": "long"
-                },
-                "os_minor": {
-                  "type": "long"
-                },
-                "os_name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "patch": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "written": {
-              "type": "long"
-            }
-          }
-        }
-      }
-    }
-  },
-  "order": 1,
-  "settings": {
-    "index": {
-      "number_of_replicas": "0",
-      "number_of_routing_shards": "30",
-      "number_of_shards": "1",
-      "refresh_interval": "5s"
-    }
-  }
-}
diff --git a/roles/system/logs/tasks/elasticsearch.yml b/roles/system/logs/tasks/elasticsearch.yml
index cd712d3..8d68b34 100644
--- a/roles/system/logs/tasks/elasticsearch.yml
+++ b/roles/system/logs/tasks/elasticsearch.yml
@@ -80,7 +80,11 @@
     dest: /srv/elasticsearch/{{ item }}-pipeline.json
     validate: "curl --fail --retry 20 --retry-connrefused -X PUT -d @%s -H 'Content-Type: application/json' http://localhost:9200/_ingest/pipeline/{{ item }}"
   loop:
-  - gitlab-workhorse
+  - gitlab-workhorse-2
+  - gitlab-workhorse-2-access
+  - gitlab-workhorse-2-msg
+  tags:
+  - elasticsearch-data
 
 - name: setup elasticsearch index template
   copy:
@@ -88,4 +92,6 @@
     dest: /srv/elasticsearch/{{ item }}-template.json
     validate: "curl --fail --retry 20 --retry-connrefused -X PUT -d @%s -H 'Content-Type: application/json' http://localhost:9200/_template/{{ item }}"
   loop:
-  - gitlab-workhorse
+  - gitlab-workhorse-2
+  tags:
+  - elasticsearch-data

    

[tor-commits] [dip/master] Make gitlab-workhorse elasticsearch 7 ready

hiro＠torproject.org