[tor-commits] [tor/master] lib: New metrics library
ahf at torproject.org
ahf at torproject.org
Tue Oct 27 15:00:48 UTC 2020
commit ec731290a5a790093961f0fdb06cf69000194adf
Author: David Goulet <dgoulet at torproject.org>
Date: Mon Oct 19 15:15:47 2020 -0400
lib: New metrics library
Used to provide an interface to create metrics store and update the entries.
Related to #40063
Signed-off-by: David Goulet <dgoulet at torproject.org>
---
Makefile.am | 6 +-
src/include.am | 3 +-
src/lib/metrics/include.am | 25 ++++++
src/lib/metrics/metrics_common.c | 29 +++++++
src/lib/metrics/metrics_common.h | 40 ++++++++++
src/lib/metrics/metrics_store.c | 140 ++++++++++++++++++++++++++++++++++
src/lib/metrics/metrics_store.h | 42 ++++++++++
src/lib/metrics/metrics_store_entry.c | 129 +++++++++++++++++++++++++++++++
src/lib/metrics/metrics_store_entry.h | 68 +++++++++++++++++
src/lib/metrics/prometheus.c | 56 ++++++++++++++
src/lib/metrics/prometheus.h | 18 +++++
11 files changed, 553 insertions(+), 3 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 50b002139e..96658230f7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -73,7 +73,8 @@ TOR_UTIL_LIBS = \
src/lib/libtor-version.a \
src/lib/libtor-llharden.a \
src/lib/libtor-intmath.a \
- src/lib/libtor-ctime.a
+ src/lib/libtor-ctime.a \
+ src/lib/libtor-metrics.a
# Variants of the above for linking the testing variant of tor (for coverage
# and tests)
@@ -108,7 +109,8 @@ TOR_UTIL_TESTING_LIBS = \
src/lib/libtor-version-testing.a \
src/lib/libtor-llharden-testing.a \
src/lib/libtor-intmath.a \
- src/lib/libtor-ctime-testing.a
+ src/lib/libtor-ctime-testing.a \
+ src/lib/libtor-metrics-testing.a
endif
# Internal crypto libraries used in Tor
diff --git a/src/include.am b/src/include.am
index 657f6e823a..95acecb057 100644
--- a/src/include.am
+++ b/src/include.am
@@ -22,10 +22,11 @@ include src/lib/intmath/include.am
include src/lib/llharden/include.am
include src/lib/lock/include.am
include src/lib/log/include.am
+include src/lib/malloc/include.am
include src/lib/math/include.am
include src/lib/memarea/include.am
include src/lib/meminfo/include.am
-include src/lib/malloc/include.am
+include src/lib/metrics/include.am
include src/lib/net/include.am
include src/lib/osinfo/include.am
include src/lib/process/include.am
diff --git a/src/lib/metrics/include.am b/src/lib/metrics/include.am
new file mode 100644
index 0000000000..62c289446e
--- /dev/null
+++ b/src/lib/metrics/include.am
@@ -0,0 +1,25 @@
+
+noinst_LIBRARIES += src/lib/libtor-metrics.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-metrics-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_metrics_a_SOURCES = \
+ src/lib/metrics/metrics_store.c \
+ src/lib/metrics/metrics_store_entry.c \
+ src/lib/metrics/metrics_common.c \
+ src/lib/metrics/prometheus.c
+
+src_lib_libtor_metrics_testing_a_SOURCES = \
+ $(src_lib_libtor_metrics_a_SOURCES)
+src_lib_libtor_metrics_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_metrics_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS += \
+ src/lib/metrics/metrics_store.h \
+ src/lib/metrics/metrics_store_entry.h \
+ src/lib/metrics/metrics_common.h \
+ src/lib/metrics/prometheus.h
diff --git a/src/lib/metrics/metrics_common.c b/src/lib/metrics/metrics_common.c
new file mode 100644
index 0000000000..5941a4d892
--- /dev/null
+++ b/src/lib/metrics/metrics_common.c
@@ -0,0 +1,29 @@
+/* 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file metrics_common.c
+ * @brief Common code for the metrics library
+ **/
+
+#include <stddef.h>
+
+#include "orconfig.h"
+
+#include "lib/log/util_bug.h"
+
+#include "lib/metrics/metrics_common.h"
+
+/** Return string representation of a metric type. */
+const char *
+metrics_type_to_str(const metrics_type_t type)
+{
+ switch (type) {
+ case METRICS_TYPE_COUNTER:
+ return "counter";
+ case METRICS_TYPE_GAUGE:
+ return "gauge";
+ default:
+ tor_assert_unreached();
+ }
+}
diff --git a/src/lib/metrics/metrics_common.h b/src/lib/metrics/metrics_common.h
new file mode 100644
index 0000000000..5d1a32ea6c
--- /dev/null
+++ b/src/lib/metrics/metrics_common.h
@@ -0,0 +1,40 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file metrics_common.h
+ * @brief Header for lib/metrics/metrics_common.c
+ **/
+
+#ifndef TOR_LIB_METRICS_METRICS_COMMON_H
+#define TOR_LIB_METRICS_METRICS_COMMON_H
+
+#include "lib/cc/torint.h"
+
+/** Format output type. */
+typedef enum {
+ /** Prometheus data output format. */
+ METRICS_FORMAT_PROMETHEUS = 1,
+} metrics_format_t;
+
+/** Metric type. */
+typedef enum {
+ /* Increment only. */
+ METRICS_TYPE_COUNTER,
+ /* Can go up or down. */
+ METRICS_TYPE_GAUGE,
+} metrics_type_t;
+
+/** Metric counter object (METRICS_TYPE_COUNTER). */
+typedef struct metrics_counter_t {
+ uint64_t value;
+} metrics_counter_t;
+
+/** Metric gauge object (METRICS_TYPE_GAUGE). */
+typedef struct metrics_gauge_t {
+ int64_t value;
+} metrics_gauge_t;
+
+const char *metrics_type_to_str(const metrics_type_t type);
+
+#endif /* !defined(TOR_LIB_METRICS_METRICS_COMMON_H) */
diff --git a/src/lib/metrics/metrics_store.c b/src/lib/metrics/metrics_store.c
new file mode 100644
index 0000000000..4f048e103d
--- /dev/null
+++ b/src/lib/metrics/metrics_store.c
@@ -0,0 +1,140 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file metrics_store.c
+ * @brief Metrics interface to store them based on specific store type and get
+ * their MetricsPort output.
+ **/
+
+#define METRICS_STORE_ENTRY_PRIVATE
+
+#include "orconfig.h"
+
+#include "lib/container/map.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+#include "lib/metrics/metrics_store.h"
+#include "lib/metrics/metrics_store_entry.h"
+
+/* Format Drivers. */
+#include "lib/metrics/prometheus.h"
+
+/** A metric store which contains a map of entries. */
+typedef struct metrics_store_t {
+ /** Indexed by metrics entry name. An entry is a smartlist_t of one or more
+ * metrics_store_entry_t allowing for multiple metrics of the same name.
+ *
+ * The reason we allow multiple entires is because there are cases where one
+ * metrics can be used twice by the same entity but with different labels.
+ * One example is an onion service with multiple ports, the port specific
+ * metrics will have a port value as a label. */
+ strmap_t *entries;
+} metrics_store_t;
+
+/** Function pointer to the format function of a specific driver. */
+typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *);
+
+/** Helper: Free a single entry in a metrics_store_t taking a void pointer
+ * parameter. */
+static void
+metrics_store_free_void(void *p)
+{
+ smartlist_t *list = p;
+ SMARTLIST_FOREACH(list, metrics_store_entry_t *, entry,
+ metrics_store_entry_free(entry));
+ smartlist_free(list);
+}
+
+/** Put the given store output in the buffer data and use the format function
+ * given in fmt to get it for each entry. */
+static void
+get_output(const metrics_store_t *store, buf_t *data, fmt_driver_fn_t fmt)
+{
+ tor_assert(store);
+ tor_assert(data);
+ tor_assert(fmt);
+
+ STRMAP_FOREACH(store->entries, key, const smartlist_t *, entries) {
+ SMARTLIST_FOREACH_BEGIN(entries, const metrics_store_entry_t *, entry) {
+ fmt(entry, data);
+ } SMARTLIST_FOREACH_END(entry);
+ } STRMAP_FOREACH_END;
+}
+
+/** Return a newly allocated and initialized store of the given type. */
+metrics_store_t *
+metrics_store_new(void)
+{
+ metrics_store_t *store = tor_malloc_zero(sizeof(*store));
+
+ store->entries = strmap_new();
+
+ return store;
+}
+
+/** Free the given store including all its entries. */
+void
+metrics_store_free_(metrics_store_t *store)
+{
+ if (store == NULL) {
+ return;
+ }
+
+ strmap_free(store->entries, metrics_store_free_void);
+ tor_free(store);
+}
+
+/** Find all metrics entry in the given store identified by name. If not found,
+ * NULL is returned. */
+smartlist_t *
+metrics_store_get_all(const metrics_store_t *store, const char *name)
+{
+ tor_assert(store);
+ tor_assert(name);
+
+ return strmap_get(store->entries, name);
+}
+
+/** Add a new metrics entry to the given store and type. The name MUST be the
+ * unique identifier. The help string can be omitted. */
+metrics_store_entry_t *
+metrics_store_add(metrics_store_t *store, metrics_type_t type,
+ const char *name, const char *help)
+{
+ smartlist_t *entries;
+ metrics_store_entry_t *entry;
+
+ tor_assert(store);
+ tor_assert(name);
+
+ entries = metrics_store_get_all(store, name);
+ if (!entries) {
+ entries = smartlist_new();
+ strmap_set(store->entries, name, entries);
+ }
+ entry = metrics_store_entry_new(type, name, help);
+ smartlist_add(entries, entry);
+
+ return entry;
+}
+
+/** Set the output of the given store of the format fmt into the given buffer
+ * data. */
+void
+metrics_store_get_output(const metrics_format_t fmt,
+ const metrics_store_t *store, buf_t *data)
+{
+ tor_assert(store);
+
+ switch (fmt) {
+ case METRICS_FORMAT_PROMETHEUS:
+ get_output(store, data, prometheus_format_store_entry);
+ break;
+ default:
+ // LCOV_EXCL_START
+ tor_assert_unreached();
+ // LCOV_EXCL_STOP
+ }
+}
diff --git a/src/lib/metrics/metrics_store.h b/src/lib/metrics/metrics_store.h
new file mode 100644
index 0000000000..9640a5e016
--- /dev/null
+++ b/src/lib/metrics/metrics_store.h
@@ -0,0 +1,42 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file metrics_store.h
+ * @brief Header for lib/metrics/metrics_store.c
+ **/
+
+#ifndef TOR_LIB_METRICS_METRICS_STORE_H
+#define TOR_LIB_METRICS_METRICS_STORE_H
+
+#include "lib/buf/buffers.h"
+#include "lib/container/smartlist.h"
+
+#include "lib/metrics/metrics_common.h"
+#include "lib/metrics/metrics_store_entry.h"
+
+/* Stub. */
+typedef struct metrics_store_t metrics_store_t;
+
+/* Allocators. */
+void metrics_store_free_(metrics_store_t *store);
+#define metrics_store_free(store) \
+ FREE_AND_NULL(metrics_store_t, metrics_store_free_, (store))
+metrics_store_t *metrics_store_new(void);
+
+/* Modifiers. */
+metrics_store_entry_t *metrics_store_add(metrics_store_t *store,
+ metrics_type_t type,
+ const char *name, const char *help);
+
+/* Accessors. */
+smartlist_t *metrics_store_get_all(const metrics_store_t *store,
+ const char *name);
+void metrics_store_get_output(const metrics_format_t fmt,
+ const metrics_store_t *store, buf_t *data);
+
+#ifdef METRICS_METRICS_STORE_PRIVATE
+
+#endif /* METRICS_METRICS_STORE_PRIVATE. */
+
+#endif /* !defined(TOR_LIB_METRICS_METRICS_STORE_H) */
diff --git a/src/lib/metrics/metrics_store_entry.c b/src/lib/metrics/metrics_store_entry.c
new file mode 100644
index 0000000000..44ebb5cb84
--- /dev/null
+++ b/src/lib/metrics/metrics_store_entry.c
@@ -0,0 +1,129 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file metrics_store_entry.c
+ * @brief Metrics store entry which contains the gathered data.
+ **/
+
+#define METRICS_STORE_ENTRY_PRIVATE
+
+#include <string.h>
+
+#include "orconfig.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+#include "lib/metrics/metrics_store_entry.h"
+
+/*
+ * Public API.
+ */
+
+/** Return newly allocated store entry of type COUNTER. */
+metrics_store_entry_t *
+metrics_store_entry_new(const metrics_type_t type, const char *name,
+ const char *help)
+{
+ metrics_store_entry_t *entry = tor_malloc_zero(sizeof(*entry));
+
+ tor_assert(name);
+
+ entry->type = type;
+ entry->name = tor_strdup(name);
+ entry->labels = smartlist_new();
+ if (help) {
+ entry->help = tor_strdup(help);
+ }
+
+ return entry;
+}
+
+/** Free a store entry. */
+void
+metrics_store_entry_free_(metrics_store_entry_t *entry)
+{
+ if (!entry) {
+ return;
+ }
+ SMARTLIST_FOREACH(entry->labels, char *, l, tor_free(l));
+ smartlist_free(entry->labels);
+ tor_free(entry->name);
+ tor_free(entry->help);
+ tor_free(entry);
+}
+
+/** Update a store entry with value. */
+void
+metrics_store_entry_update(metrics_store_entry_t *entry, const int64_t value)
+{
+ tor_assert(entry);
+
+ switch (entry->type) {
+ case METRICS_TYPE_COUNTER:
+ /* Counter can ONLY be positive. */
+ if (BUG(value < 0)) {
+ return;
+ }
+ entry->u.counter.value += value;
+ break;
+ case METRICS_TYPE_GAUGE:
+ /* Gauge can increment or decrement. And can be positive or negative. */
+ entry->u.gauge.value += value;
+ break;
+ }
+}
+
+/** Reset a store entry that is set its metric data to 0. */
+void
+metrics_store_entry_reset(metrics_store_entry_t *entry)
+{
+ tor_assert(entry);
+ /* Everything back to 0. */
+ memset(&entry->u, 0, sizeof(entry->u));
+}
+
+/** Return store entry value. */
+int64_t
+metrics_store_entry_get_value(const metrics_store_entry_t *entry)
+{
+ tor_assert(entry);
+
+ switch (entry->type) {
+ case METRICS_TYPE_COUNTER:
+ if (entry->u.counter.value > INT64_MAX) {
+ return INT64_MAX;
+ }
+ return entry->u.counter.value;
+ case METRICS_TYPE_GAUGE:
+ return entry->u.gauge.value;
+ }
+
+ // LCOV_EXCL_START
+ tor_assert_unreached();
+ // LCOV_EXCL_STOP
+}
+
+/** Add a label into the given entry.*/
+void
+metrics_store_entry_add_label(metrics_store_entry_t *entry,
+ const char *label)
+{
+ tor_assert(entry);
+ tor_assert(label);
+
+ smartlist_add(entry->labels, tor_strdup(label));
+}
+
+/** Return true iff the given entry has the given label. */
+bool
+metrics_store_entry_has_label(const metrics_store_entry_t *entry,
+ const char *label)
+{
+ tor_assert(entry);
+ tor_assert(label);
+
+ return smartlist_contains_string(entry->labels, label);
+}
diff --git a/src/lib/metrics/metrics_store_entry.h b/src/lib/metrics/metrics_store_entry.h
new file mode 100644
index 0000000000..6fff9d10eb
--- /dev/null
+++ b/src/lib/metrics/metrics_store_entry.h
@@ -0,0 +1,68 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file metrics_store_entry.h
+ * @brief Header for lib/metrics/metrics_store_entry.c
+ **/
+
+#ifndef TOR_LIB_METRICS_METRICS_STORE_ENTRY_H
+#define TOR_LIB_METRICS_METRICS_STORE_ENTRY_H
+
+#include "lib/cc/torint.h"
+
+#include "lib/metrics/metrics_common.h"
+
+#ifdef METRICS_STORE_ENTRY_PRIVATE
+
+/** Metrics store entry. They reside in a metrics_store_t object and are
+ * opaque to the outside world. */
+typedef struct metrics_store_entry_t {
+ /** Type of entry. */
+ metrics_type_t type;
+
+ /** Name. */
+ char *name;
+
+ /** Help comment string. */
+ char *help;
+
+ /** Labels attached to that entry. If NULL, no labels.
+ *
+ * Labels are used to add extra context to a metrics. For example, a label
+ * could be an onion address so the metrics can be differentiate. */
+ smartlist_t *labels;
+
+ /* Actual data. */
+ union {
+ metrics_counter_t counter;
+ metrics_gauge_t gauge;
+ } u;
+} metrics_store_entry_t;
+
+#endif /* METRICS_STORE_ENTRY_PRIVATE */
+
+typedef struct metrics_store_entry_t metrics_store_entry_t;
+
+/* Allocators. */
+metrics_store_entry_t *metrics_store_entry_new(const metrics_type_t type,
+ const char *name,
+ const char *help);
+
+void metrics_store_entry_free_(metrics_store_entry_t *entry);
+#define metrics_store_entry_free(entry) \
+ FREE_AND_NULL(metrics_store_entry_t, metrics_store_entry_free_, (entry));
+
+/* Accessors. */
+int64_t metrics_store_entry_get_value(const metrics_store_entry_t *entry);
+bool metrics_store_entry_has_label(const metrics_store_entry_t *entry,
+ const char *label);
+
+/* Modifiers. */
+void metrics_store_entry_add_label(metrics_store_entry_t *entry,
+ const char *label);
+void metrics_store_entry_reset(metrics_store_entry_t *entry);
+void metrics_store_entry_update(metrics_store_entry_t *entry,
+ const int64_t value);
+
+#endif /* !defined(TOR_LIB_METRICS_METRICS_STORE_ENTRY_H) */
diff --git a/src/lib/metrics/prometheus.c b/src/lib/metrics/prometheus.c
new file mode 100644
index 0000000000..c2b54e436f
--- /dev/null
+++ b/src/lib/metrics/prometheus.c
@@ -0,0 +1,56 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file prometheus.c
+ * @brief Metrics format driver for Prometheus data model.
+ **/
+
+#define METRICS_STORE_ENTRY_PRIVATE
+
+#include "orconfig.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "lib/metrics/prometheus.h"
+
+/** Return a static buffer containing all the labels properly formatted
+ * for the output as a string.
+ *
+ * Subsequent calls to this invalidates the previous result. */
+static const char *
+format_labels(smartlist_t *labels)
+{
+ static char buf[1024];
+ char *line = NULL;
+
+ if (smartlist_len(labels) == 0) {
+ buf[0] = '\0';
+ goto end;
+ }
+
+ line = smartlist_join_strings(labels, ",", 0, NULL);
+ tor_snprintf(buf, sizeof(buf), "{%s}", line);
+
+ end:
+ tor_free(line);
+ return buf;
+}
+
+/** Format the given entry in to the buffer data. */
+void
+prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data)
+{
+ tor_assert(entry);
+ tor_assert(data);
+
+ buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help);
+ buf_add_printf(data, "# TYPE %s %s\n", entry->name,
+ metrics_type_to_str(entry->type));
+ buf_add_printf(data, "%s%s %" PRIi64 "\n", entry->name,
+ format_labels(entry->labels),
+ metrics_store_entry_get_value(entry));
+}
diff --git a/src/lib/metrics/prometheus.h b/src/lib/metrics/prometheus.h
new file mode 100644
index 0000000000..eea26e8ac4
--- /dev/null
+++ b/src/lib/metrics/prometheus.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file prometheus.h
+ * @brief Header for feature/metrics/prometheus.c
+ **/
+
+#ifndef TOR_LIB_METRICS_PROMETHEUS_H
+#define TOR_LIB_METRICS_PROMETHEUS_H
+
+#include "lib/buf/buffers.h"
+#include "lib/metrics/metrics_store_entry.h"
+
+void prometheus_format_store_entry(const metrics_store_entry_t *entry,
+ buf_t *data);
+
+#endif /* !defined(TOR_LIB_METRICS_PROMETHEUS_H) */
More information about the tor-commits
mailing list