tor-commits October 2020

tor-commits@lists.torproject.org

17 participants
2274 discussions

[tor/master] doc: Initial documentation of the lib/metrics
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit 1c06c9f1c0a496132a586ab8ff6ecf371b6ecf43 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Oct 22 15:11:58 2020 -0400 doc: Initial documentation of the lib/metrics Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/lib/metrics/lib_metrics.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/lib/metrics/lib_metrics.md b/src/lib/metrics/lib_metrics.md new file mode 100644 index 0000000000..e58680e237 --- /dev/null +++ b/src/lib/metrics/lib_metrics.md @@ -0,0 +1,12 @@ +@dir /lib/metrics +@brief lib/metrics: Metrics collection API + +This module is used for adding "metrics" support to Tor. + +Metrics are a collection of counters that are defined per-subsystem and +accessed through the MetricsPort. Each subsystem is responsible for populating +metrics store(s) and providing access to them through the `.get_metrics()` +call located in the `subsys_fns_t` object. + +These metrics are meant to be extremely lightweight and thus can be accessed +without too much CPU cost.

1 0

[tor/master] hs: Collect rendezvous circuit metrics
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit c081bee38a428dda2bd9ee043f7cbab3cbd77393 Author: David Goulet <dgoulet(a)torproject.org> Date: Tue Oct 20 14:59:44 2020 -0400 hs: Collect rendezvous circuit metrics The total number of rendezvous circuit created and the number of established ones which is a gauge that decreases to keep an updated counter. Related to #40063 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/feature/hs/hs_circuit.c | 10 ++++++++++ src/feature/hs/hs_metrics.h | 14 ++++++++++++++ src/feature/hs/hs_metrics_entry.c | 12 ++++++++++++ src/feature/hs/hs_metrics_entry.h | 4 ++++ src/feature/hs/hs_service.c | 29 +++++++++++++++++++++++++++++ src/feature/hs/hs_service.h | 1 + 6 files changed, 70 insertions(+) diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 5e7a3ac9c8..e4422c86a8 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -24,6 +24,7 @@ #include "feature/hs/hs_circuitmap.h" #include "feature/hs/hs_client.h" #include "feature/hs/hs_ident.h" +#include "feature/hs/hs_metrics.h" #include "feature/hs/hs_service.h" #include "feature/nodelist/describe.h" #include "feature/nodelist/nodelist.h" @@ -429,6 +430,9 @@ launch_rendezvous_point_circuit,(const hs_service_t *service, safe_str_client(service->onion_address)); goto end; } + /* Update metrics with this new rendezvous circuit launched. */ + hs_metrics_new_rdv(&service->keys.identity_pk); + log_info(LD_REND, "Rendezvous circuit launched to %s with cookie %s " "for %s service %s", safe_str_client(extend_info_describe(info)), @@ -1312,6 +1316,12 @@ hs_circ_cleanup_on_close(circuit_t *circ) cleanup_on_close_client_circ(circ); } + if (circuit_purpose_is_hs_service(circ->purpose)) { + if (circuit_is_hs_v3(circ)) { + hs_service_circuit_cleanup_on_close(circ); + } + } + /* On close, we simply remove it from the circuit map. It can not be used * anymore. We keep this code path fast and lean. */ diff --git a/src/feature/hs/hs_metrics.h b/src/feature/hs/hs_metrics.h index 76eb68a2b1..21af42e01c 100644 --- a/src/feature/hs/hs_metrics.h +++ b/src/feature/hs/hs_metrics.h @@ -43,4 +43,18 @@ void hs_metrics_update_by_service(const hs_metrics_key_t key, #define hs_metrics_app_read_bytes(i, port, n) \ hs_metrics_update_by_ident(HS_METRICS_APP_READ_BYTES, (i), (port), (n)) +/** Newly established rendezvous. This is called as soon as the circuit purpose + * is REND_JOINED which is when the RENDEZVOUS2 cell is sent. */ +#define hs_metrics_new_established_rdv(s) \ + hs_metrics_update_by_service(HS_METRICS_NUM_ESTABLISHED_RDV, (s), 0, 1) + +/** Established rendezvous closed. This is called when the circuit in + * REND_JOINED state is marked for close. */ +#define hs_metrics_close_established_rdv(i) \ + hs_metrics_update_by_ident(HS_METRICS_NUM_ESTABLISHED_RDV, (i), 0, -1) + +/** New rendezvous circuit being launched. */ +#define hs_metrics_new_rdv(i) \ + hs_metrics_update_by_ident(HS_METRICS_NUM_RDV, (i), 0, 1) + #endif /* !defined(TOR_FEATURE_HS_HS_METRICS_H) */ diff --git a/src/feature/hs/hs_metrics_entry.c b/src/feature/hs/hs_metrics_entry.c index 6d0a143503..78f5253327 100644 --- a/src/feature/hs/hs_metrics_entry.c +++ b/src/feature/hs/hs_metrics_entry.c @@ -41,6 +41,18 @@ const hs_metrics_entry_t base_metrics[] = .help = "Total number of bytes read from the application", .port_as_label = true, }, + { + .key = HS_METRICS_NUM_ESTABLISHED_RDV, + .type = METRICS_TYPE_GAUGE, + .name = "hs_rdv_established_count", + .help = "Total number of established rendezvous circuit", + }, + { + .key = HS_METRICS_NUM_RDV, + .type = METRICS_TYPE_COUNTER, + .name = "hs_rdv_num_total", + .help = "Total number of rendezvous circuit created", + }, }; /** Size of base_metrics array that is number of entries. */ diff --git a/src/feature/hs/hs_metrics_entry.h b/src/feature/hs/hs_metrics_entry.h index e4b999fc46..924696df4f 100644 --- a/src/feature/hs/hs_metrics_entry.h +++ b/src/feature/hs/hs_metrics_entry.h @@ -21,6 +21,10 @@ typedef enum { HS_METRICS_APP_WRITE_BYTES = 1, /** Number of bytes read from application to onion service. */ HS_METRICS_APP_READ_BYTES = 2, + /** Number of established rendezsvous. */ + HS_METRICS_NUM_ESTABLISHED_RDV = 3, + /** Number of rendezsvous circuits created. */ + HS_METRICS_NUM_RDV = 4, } hs_metrics_key_t; /** The metadata of an HS metrics. */ diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 1ccd3e4435..6cf5257eb9 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -3391,6 +3391,15 @@ service_rendezvous_circ_has_opened(origin_circuit_t *circ) /* If the cell can't be sent, the circuit will be closed within this * function. */ hs_circ_service_rp_has_opened(service, circ); + + /* Update metrics that we have an established rendezvous circuit. It is not + * entirely true until the client receives the RENDEZVOUS2 cell and starts + * sending but if that circuit collapes, we'll decrement the counter thus it + * will even out the metric. */ + if (TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) { + hs_metrics_new_established_rdv(service); + } + goto done; err: @@ -3579,6 +3588,26 @@ service_encode_descriptor(const hs_service_t *service, /* Public API */ /* ========== */ +/** Called when a circuit was just cleaned up. This is done right before the + * circuit is marked for close. */ +void +hs_service_circuit_cleanup_on_close(const circuit_t *circ) +{ + tor_assert(circ); + tor_assert(CIRCUIT_IS_ORIGIN(circ)); + + switch (circ->purpose) { + case CIRCUIT_PURPOSE_S_REND_JOINED: + /* About to close an established rendezvous circuit. Update the metrics to + * reflect how many we have at the moment. */ + hs_metrics_close_established_rdv( + &CONST_TO_ORIGIN_CIRCUIT(circ)->hs_ident->identity_pk); + break; + default: + break; + } +} + /** This is called everytime the service map (v2 or v3) changes that is if an * element is added or removed. */ void diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index 136ff744db..974f46b960 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -386,6 +386,7 @@ hs_circuit_id_protocol_t hs_service_exports_circuit_id(const ed25519_public_key_t *pk); void hs_service_dump_stats(int severity); +void hs_service_circuit_cleanup_on_close(const circuit_t *circ); #ifdef HS_SERVICE_PRIVATE

1 0

[tor/master] hs: Collect service traffic metrics
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit 2fe0322fe4550e59cb8757fbff6984286bf8e121 Author: David Goulet <dgoulet(a)torproject.org> Date: Tue Oct 20 14:47:05 2020 -0400 hs: Collect service traffic metrics Related to #40063 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/core/mainloop/connection.c | 9 +++++++++ src/core/or/relay.c | 8 ++++++++ src/feature/hs/hs_metrics.h | 8 ++++++++ src/feature/hs/hs_metrics_entry.c | 14 ++++++++++++++ src/feature/hs/hs_metrics_entry.h | 4 ++++ 5 files changed, 43 insertions(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index ebf15fcc9e..3d551c4ba8 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -99,6 +99,7 @@ #include "feature/hibernate/hibernate.h" #include "feature/hs/hs_common.h" #include "feature/hs/hs_ident.h" +#include "feature/hs/hs_metrics.h" #include "feature/metrics/metrics.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" @@ -4149,6 +4150,14 @@ connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read, /* change *max_to_read */ *max_to_read = at_most - n_read; + /* Onion service application connection. Note read bytes for metrics. */ + if (CONN_IS_EDGE(conn) && TO_EDGE_CONN(conn)->hs_ident) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + hs_metrics_app_read_bytes(&edge_conn->hs_ident->identity_pk, + edge_conn->hs_ident->orig_virtual_port, + n_read); + } + /* Update edge_conn->n_read */ if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 6895591064..f986883370 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -83,6 +83,7 @@ #include "feature/nodelist/describe.h" #include "feature/nodelist/routerlist.h" #include "core/or/scheduler.h" +#include "feature/hs/hs_metrics.h" #include "core/or/cell_st.h" #include "core/or/cell_queue_st.h" @@ -1689,6 +1690,13 @@ handle_relay_cell_command(cell_t *cell, circuit_t *circ, circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); } + /* For onion service connection, update the metrics. */ + if (conn->hs_ident) { + hs_metrics_app_write_bytes(&conn->hs_ident->identity_pk, + conn->hs_ident->orig_virtual_port, + rh->length); + } + stats_n_data_bytes_received += rh->length; connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE), rh->length, TO_CONN(conn)); diff --git a/src/feature/hs/hs_metrics.h b/src/feature/hs/hs_metrics.h index 991b66a7ee..76eb68a2b1 100644 --- a/src/feature/hs/hs_metrics.h +++ b/src/feature/hs/hs_metrics.h @@ -35,4 +35,12 @@ void hs_metrics_update_by_service(const hs_metrics_key_t key, #define hs_metrics_new_introduction(s) \ hs_metrics_update_by_service(HS_METRICS_NUM_INTRODUCTIONS, (s), 0, 1) +/** Number of bytes written to the application from the service. */ +#define hs_metrics_app_write_bytes(i, port, n) \ + hs_metrics_update_by_ident(HS_METRICS_APP_WRITE_BYTES, (i), (port), (n)) + +/** Number of bytes read from the application to the service. */ +#define hs_metrics_app_read_bytes(i, port, n) \ + hs_metrics_update_by_ident(HS_METRICS_APP_READ_BYTES, (i), (port), (n)) + #endif /* !defined(TOR_FEATURE_HS_HS_METRICS_H) */ diff --git a/src/feature/hs/hs_metrics_entry.c b/src/feature/hs/hs_metrics_entry.c index d6b2e0e62b..6d0a143503 100644 --- a/src/feature/hs/hs_metrics_entry.c +++ b/src/feature/hs/hs_metrics_entry.c @@ -27,6 +27,20 @@ const hs_metrics_entry_t base_metrics[] = .help = "Total number of introduction received", .port_as_label = false, }, + { + .key = HS_METRICS_APP_WRITE_BYTES, + .type = METRICS_TYPE_COUNTER, + .name = "hs_app_write_bytes_total", + .help = "Total number of bytes written to the application", + .port_as_label = true, + }, + { + .key = HS_METRICS_APP_READ_BYTES, + .type = METRICS_TYPE_COUNTER, + .name = "hs_app_read_bytes_total", + .help = "Total number of bytes read from the application", + .port_as_label = true, + }, }; /** Size of base_metrics array that is number of entries. */ diff --git a/src/feature/hs/hs_metrics_entry.h b/src/feature/hs/hs_metrics_entry.h index 96dce36ffa..e4b999fc46 100644 --- a/src/feature/hs/hs_metrics_entry.h +++ b/src/feature/hs/hs_metrics_entry.h @@ -17,6 +17,10 @@ typedef enum { /** Number of introduction requests. */ HS_METRICS_NUM_INTRODUCTIONS = 0, + /** Number of bytes written from onion service to application. */ + HS_METRICS_APP_WRITE_BYTES = 1, + /** Number of bytes read from application to onion service. */ + HS_METRICS_APP_READ_BYTES = 2, } hs_metrics_key_t; /** The metadata of an HS metrics. */

1 0

[tor/master] man: Add the MetricsPort manpage entry
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit 04ff22dc69496f8b05e9ef9ea3a0216ca5fec5be Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Oct 22 14:21:43 2020 -0400 man: Add the MetricsPort manpage entry Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- doc/man/tor.1.txt | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/doc/man/tor.1.txt b/doc/man/tor.1.txt index 5f94f414b8..ab273925b1 100644 --- a/doc/man/tor.1.txt +++ b/doc/man/tor.1.txt @@ -744,6 +744,48 @@ forward slash (/) in the configuration file and on the command line. total; this is intended to be used to debug problems without opening live servers to resource exhaustion attacks. (Default: 10 MBytes) +[[MetricsPort]] **MetricsPort** ['address'**:**]{empty}__port__ [__format__]:: + WARNING: Before enabling this, it is important to understand that exposing + tor metrics publicly is dangerous to the Tor network users. Please take + extra precaution and care when opening this port. Set a very strict access + policy with MetricsPortPolicy and consider using your operating systems + firewall features for defense in depth. + + + We recommend, for the prometheus __format__, that the only address that + can access this port should be the Prometheus server itself. Remember that + the connection is unencrypted (HTTP) hence consider using a tool like + stunnel to secure the link from this port to the server. + + + If set, open this port to listen for an HTTP GET request to "/metrics". + Upon a request, the collected metrics in the the tor instance are + formatted for the given format and then sent back. If this is set, + MetricsPortPolicy must be defined else every request will be rejected. + + + Supported format is "prometheus" which is also the default if not set. The + Prometheus data model can be found here: + https://prometheus.io/docs/concepts/data_model/ + + + The tor metrics are constantly collected and they solely consists of + counters. Thus, asking for those metrics is very lightweight on the tor + process. (Default: None) + + + As an example, here only 5.6.7.8 will be allowed to connect: + + MetricsPort 1.2.3.4:9035 + MetricsPortPolicy accept 5.6.7.8 + +[[MetricsPortPolicy]] **MetricsPortPolicy** __policy__,__policy__,__...__:: + Set an entrance policy for the **MetricsPort**, to limit who can access + it. The policies have the same form as exit policies below, except that + port specifiers are ignored. For multiple entries, this line can be used + multiple times. It is a reject all by default policy. (Default: None) + + + Please, keep in mind here that if the server collecting metrics on the + MetricsPort is behind a NAT, then everything behind it can access it. This + is similar for the case of allowing localhost, every users on the server + will be able to access it. Again, strongly consider using a tool like + stunnel to secure the link or to strengthen access control. + [[NoExec]] **NoExec** **0**|**1**:: If this option is set to 1, then Tor will never launch another executable, regardless of the settings of ClientTransportPlugin

1 0

[tor/master] metrics: Add tor_ namespace to all metrics
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit 5de66739724451801487dbec0eb210c0595cc00f Author: David Goulet <dgoulet(a)torproject.org> Date: Tue Oct 27 10:43:19 2020 -0400 metrics: Add tor_ namespace to all metrics Closes #40063 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/feature/hs/hs_metrics_entry.c | 12 ++++++------ src/lib/metrics/metrics_common.h | 9 +++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/feature/hs/hs_metrics_entry.c b/src/feature/hs/hs_metrics_entry.c index e4da0921aa..7eb78db5ac 100644 --- a/src/feature/hs/hs_metrics_entry.c +++ b/src/feature/hs/hs_metrics_entry.c @@ -23,40 +23,40 @@ const hs_metrics_entry_t base_metrics[] = { .key = HS_METRICS_NUM_INTRODUCTIONS, .type = METRICS_TYPE_COUNTER, - .name = "hs_intro_num_total", + .name = METRICS_NAME(hs_intro_num_total), .help = "Total number of introduction received", .port_as_label = false, }, { .key = HS_METRICS_APP_WRITE_BYTES, .type = METRICS_TYPE_COUNTER, - .name = "hs_app_write_bytes_total", + .name = METRICS_NAME(hs_app_write_bytes_total), .help = "Total number of bytes written to the application", .port_as_label = true, }, { .key = HS_METRICS_APP_READ_BYTES, .type = METRICS_TYPE_COUNTER, - .name = "hs_app_read_bytes_total", + .name = METRICS_NAME(hs_app_read_bytes_total), .help = "Total number of bytes read from the application", .port_as_label = true, }, { .key = HS_METRICS_NUM_ESTABLISHED_RDV, .type = METRICS_TYPE_GAUGE, - .name = "hs_rdv_established_count", + .name = METRICS_NAME(hs_rdv_established_count), .help = "Total number of established rendezvous circuit", }, { .key = HS_METRICS_NUM_RDV, .type = METRICS_TYPE_COUNTER, - .name = "hs_rdv_num_total", + .name = METRICS_NAME(hs_rdv_num_total), .help = "Total number of rendezvous circuit created", }, { .key = HS_METRICS_NUM_ESTABLISHED_INTRO, .type = METRICS_TYPE_GAUGE, - .name = "hs_intro_established_count", + .name = METRICS_NAME(hs_intro_established_count), .help = "Total number of established introduction circuit", }, }; diff --git a/src/lib/metrics/metrics_common.h b/src/lib/metrics/metrics_common.h index 5d1a32ea6c..50f6147007 100644 --- a/src/lib/metrics/metrics_common.h +++ b/src/lib/metrics/metrics_common.h @@ -11,6 +11,15 @@ #include "lib/cc/torint.h" +/** Prefix to every metrics exposed. This is insures that the metrics are + * always in the same namespace. */ +#define METRICS_PREFIX tor_ + +/** Helper macro that must be used to construct the right namespaced metrics + * name. A name is a string so stringify the result. */ +#define METRICS_STR(val) #val +#define METRICS_NAME(name) METRICS_STR(METRICS_PREFIX ## name) + /** Format output type. */ typedef enum { /** Prometheus data output format. */

1 0

[tor/master] changes: File for ticket 40063, the MetricsPort
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit f2e42045902524cb8277ab9674f9d669d2d8ab00 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Oct 22 15:15:33 2020 -0400 changes: File for ticket 40063, the MetricsPort Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket40063 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changes/ticket40063 b/changes/ticket40063 new file mode 100644 index 0000000000..ddf1349044 --- /dev/null +++ b/changes/ticket40063 @@ -0,0 +1,5 @@ + o Major features (metrics): + - Introduce a new MetricsPort which exposes, through an HTTP GET /metrics, a + series of metrics that tor collects at runtime. At the moment, the only + supported output format is Prometheus data model. Closes ticket 40063; +

1 0

[tor/master] test: Metrics tests for lib/ and feature/
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit f65c0820af6082f15541756eaaeb042716751098 Author: David Goulet <dgoulet(a)torproject.org> Date: Fri Oct 23 12:27:12 2020 -0400 test: Metrics tests for lib/ and feature/ Related to #40063 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_metrics.c | 254 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 257 insertions(+) diff --git a/src/test/include.am b/src/test/include.am index cb3a498f74..5149416044 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -200,6 +200,7 @@ src_test_test_SOURCES += \ src/test/test_link_handshake.c \ src/test/test_logging.c \ src/test/test_mainloop.c \ + src/test/test_metrics.c \ src/test/test_microdesc.c \ src/test/test_namemap.c \ src/test/test_netinfo.c \ diff --git a/src/test/test.c b/src/test/test.c index 77aa6db975..9e0fc4433b 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -738,6 +738,7 @@ struct testgroup_t testgroups[] = { { "legacy_hs/", hs_tests }, { "link-handshake/", link_handshake_tests }, { "mainloop/", mainloop_tests }, + { "metrics/", metrics_tests }, { "netinfo/", netinfo_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, diff --git a/src/test/test.h b/src/test/test.h index bd3a4102f5..ba1d69fa11 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -150,6 +150,7 @@ extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; extern struct testcase_t logging_tests[]; extern struct testcase_t mainloop_tests[]; +extern struct testcase_t metrics_tests[]; extern struct testcase_t microdesc_tests[]; extern struct testcase_t namemap_tests[]; extern struct testcase_t netinfo_tests[]; diff --git a/src/test/test_metrics.c b/src/test/test_metrics.c new file mode 100644 index 0000000000..26e84a5798 --- /dev/null +++ b/src/test/test_metrics.c @@ -0,0 +1,254 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_metrics.c + * \brief Test lib/metrics and feature/metrics functionnalities + */ + +#define CONFIG_PRIVATE +#define CONNECTION_PRIVATE +#define METRICS_STORE_ENTRY_PRIVATE + +#include "test/test.h" +#include "test/test_helpers.h" +#include "test/log_test_helpers.h" + +#include "app/config/config.h" + +#include "core/mainloop/connection.h" +#include "core/or/connection_st.h" +#include "core/or/policies.h" +#include "core/or/port_cfg_st.h" + +#include "feature/metrics/metrics.h" + +#include "lib/encoding/confline.h" +#include "lib/metrics/metrics_store.h" + +#define TEST_METRICS_ENTRY_NAME "entryA" +#define TEST_METRICS_ENTRY_HELP "Description of entryA" +#define TEST_METRICS_ENTRY_LABEL_1 "label=farfadet" +#define TEST_METRICS_ENTRY_LABEL_2 "label=ponki" + +static void +set_metrics_port(or_options_t *options) +{ + const char *port = "MetricsPort 9035"; /* Default to 127.0.0.1 */ + const char *policy = "MetricsPortPolicy accept 1.2.3.4"; + + config_get_lines(port, &options->MetricsPort_lines, 0); + config_get_lines(policy, &options->MetricsPortPolicy, 0); + + /* Parse and validate policy. */ + policies_parse_from_options(options); +} + +static void +test_config(void *arg) +{ + char *err_msg = NULL; + tor_addr_t addr; + smartlist_t *ports = smartlist_new(); + or_options_t *options = get_options_mutable(); + + (void) arg; + + set_metrics_port(options); + + int ret = metrics_parse_ports(options, ports, &err_msg); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(smartlist_len(ports), OP_EQ, 1); + + /* Validate the configured port. */ + const port_cfg_t *cfg = smartlist_get(ports, 0); + tt_assert(tor_addr_eq_ipv4h(&cfg->addr, 0x7f000001)); + tt_int_op(cfg->port, OP_EQ, 9035); + tt_int_op(cfg->type, OP_EQ, CONN_TYPE_METRICS_LISTENER); + + /* Address of the policy should be permitted. */ + tor_addr_from_ipv4h(&addr, 0x01020304); /* 1.2.3.4 */ + ret = metrics_policy_permits_address(&addr); + tt_int_op(ret, OP_EQ, true); + + /* Anything else, should not. */ + tor_addr_from_ipv4h(&addr, 0x01020305); /* 1.2.3.5 */ + ret = metrics_policy_permits_address(&addr); + tt_int_op(ret, OP_EQ, false); + + done: + SMARTLIST_FOREACH(ports, port_cfg_t *, c, port_cfg_free(c)); + smartlist_free(ports); + or_options_free(options); +} + +static char _c_buf[256]; +#define CONTAINS(conn, msg) \ + do { \ + tt_int_op(buf_datalen(conn->outbuf), OP_EQ, (strlen(msg))); \ + memset(_c_buf, 0, sizeof(_c_buf)); \ + buf_get_bytes(conn->outbuf, _c_buf, (strlen(msg))); \ + tt_str_op(_c_buf, OP_EQ, (msg)); \ + tt_int_op(buf_datalen(conn->outbuf), OP_EQ, 0); \ + } while (0) + +#define WRITE(conn, msg) \ + buf_add(conn->inbuf, (msg), (strlen(msg))); + +static void +test_connection(void *arg) +{ + int ret; + connection_t *conn = connection_new(CONN_TYPE_METRICS, AF_INET); + or_options_t *options = get_options_mutable(); + + (void) arg; + + /* Setup policy. */ + set_metrics_port(options); + + /* Set 1.2.3.5 IP, we should get rejected. */ + tor_addr_from_ipv4h(&conn->addr, 0x01020305); + ret = metrics_connection_process_inbuf(conn); + tt_int_op(ret, OP_EQ, -1); + + /* Set 1.2.3.4 so from now on we are allowed to process the inbuf. */ + tor_addr_from_ipv4h(&conn->addr, 0x01020304); + + /* No HTTP request yet. */ + ret = metrics_connection_process_inbuf(conn); + tt_int_op(ret, OP_EQ, -1); + + /* Bad request. */ + WRITE(conn, "HTTP 4.7\r\n\r\n"); + ret = metrics_connection_process_inbuf(conn); + tt_int_op(ret, OP_EQ, -1); + CONTAINS(conn, "HTTP/1.0 400 Bad Request\r\n\r\n"); + + /* Path not found. */ + WRITE(conn, "GET /badpath HTTP/1.0\r\n\r\n"); + ret = metrics_connection_process_inbuf(conn); + tt_int_op(ret, OP_EQ, -1); + CONTAINS(conn, "HTTP/1.0 404 Not Found\r\n\r\n"); + + /* Method not allowed. */ + WRITE(conn, "POST /something HTTP/1.0\r\n\r\n"); + ret = metrics_connection_process_inbuf(conn); + tt_int_op(ret, OP_EQ, -1); + CONTAINS(conn, "HTTP/1.0 405 Method Not Allowed\r\n\r\n"); + + /* Ask for metrics. The content should be above 0. We don't test the + * validity of the returned content but it is certainly not an error. */ + WRITE(conn, "GET /metrics HTTP/1.0\r\n\r\n"); + ret = metrics_connection_process_inbuf(conn); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(buf_datalen(conn->outbuf), OP_GT, 0); + + done: + or_options_free(options); + connection_free_minimal(conn); +} + +static void +test_prometheus(void *arg) +{ + metrics_store_t *store = NULL; + metrics_store_entry_t *entry = NULL; + buf_t *buf = buf_new(); + char *output = NULL; + + (void) arg; + + /* Fresh new store. No entries. */ + store = metrics_store_new(); + tt_assert(store); + + /* Add entry and validate its content. */ + entry = metrics_store_add(store, METRICS_TYPE_COUNTER, + TEST_METRICS_ENTRY_NAME, + TEST_METRICS_ENTRY_HELP); + tt_assert(entry); + metrics_store_entry_add_label(entry, TEST_METRICS_ENTRY_LABEL_1); + + static const char *expected = + "# HELP " TEST_METRICS_ENTRY_NAME " " TEST_METRICS_ENTRY_HELP "\n" + "# TYPE " TEST_METRICS_ENTRY_NAME " counter\n" + TEST_METRICS_ENTRY_NAME "{" TEST_METRICS_ENTRY_LABEL_1 "} 0\n"; + + metrics_store_get_output(METRICS_FORMAT_PROMETHEUS, store, buf); + output = buf_extract(buf, NULL); + tt_str_op(expected, OP_EQ, output); + + done: + buf_free(buf); + tor_free(output); + metrics_store_free(store); +} + +static void +test_store(void *arg) +{ + metrics_store_t *store = NULL; + metrics_store_entry_t *entry = NULL; + + (void) arg; + + /* Fresh new store. No entries. */ + store = metrics_store_new(); + tt_assert(store); + tt_assert(!metrics_store_get_all(store, TEST_METRICS_ENTRY_NAME)); + + /* Add entry and validate its content. */ + entry = metrics_store_add(store, METRICS_TYPE_COUNTER, + TEST_METRICS_ENTRY_NAME, + TEST_METRICS_ENTRY_HELP); + tt_assert(entry); + tt_int_op(entry->type, OP_EQ, METRICS_TYPE_COUNTER); + tt_str_op(entry->name, OP_EQ, TEST_METRICS_ENTRY_NAME); + tt_str_op(entry->help, OP_EQ, TEST_METRICS_ENTRY_HELP); + tt_uint_op(entry->u.counter.value, OP_EQ, 0); + + /* Access the entry. */ + tt_assert(metrics_store_get_all(store, TEST_METRICS_ENTRY_NAME)); + + /* Add a label to the entry to make it unique. */ + metrics_store_entry_add_label(entry, TEST_METRICS_ENTRY_LABEL_1); + tt_int_op(metrics_store_entry_has_label(entry, TEST_METRICS_ENTRY_LABEL_1), + OP_EQ, true); + + /* Update entry's value. */ + metrics_store_entry_update(entry, 42); + tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 42); + metrics_store_entry_update(entry, 42); + tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 84); + metrics_store_entry_reset(entry); + tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 0); + + /* Add a new entry of same name but different label. */ + /* Add entry and validate its content. */ + entry = metrics_store_add(store, METRICS_TYPE_COUNTER, + TEST_METRICS_ENTRY_NAME, + TEST_METRICS_ENTRY_HELP); + tt_assert(entry); + metrics_store_entry_add_label(entry, TEST_METRICS_ENTRY_LABEL_2); + + /* Make sure _both_ entires are there. */ + const smartlist_t *entries = + metrics_store_get_all(store, TEST_METRICS_ENTRY_NAME); + tt_assert(entries); + tt_int_op(smartlist_len(entries), OP_EQ, 2); + + done: + metrics_store_free(store); +} + +struct testcase_t metrics_tests[] = { + + { "config", test_config, TT_FORK, NULL, NULL }, + { "connection", test_connection, TT_FORK, NULL, NULL }, + { "prometheus", test_prometheus, TT_FORK, NULL, NULL }, + { "store", test_store, TT_FORK, NULL, NULL }, + + END_OF_TESTCASES +}; +

1 0

[tor/master] test: Add test for onion service metrics module
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit 238340ce54d1e0b2b7ef593697b971e951d9bd23 Author: David Goulet <dgoulet(a)torproject.org> Date: Fri Oct 23 13:19:38 2020 -0400 test: Add test for onion service metrics module Related to #40063 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_hs_metrics.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/src/test/include.am b/src/test/include.am index 5149416044..cdf3b20c48 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -195,6 +195,7 @@ src_test_test_SOURCES += \ src/test/test_hs_cache.c \ src/test/test_hs_descriptor.c \ src/test/test_hs_dos.c \ + src/test/test_hs_metrics.c \ src/test/test_introduce.c \ src/test/test_keypin.c \ src/test/test_link_handshake.c \ diff --git a/src/test/test.c b/src/test/test.c index 9e0fc4433b..ffea158141 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -730,6 +730,7 @@ struct testgroup_t testgroups[] = { { "hs_descriptor/", hs_descriptor }, { "hs_dos/", hs_dos_tests }, { "hs_intropoint/", hs_intropoint_tests }, + { "hs_metrics/", hs_metrics_tests }, { "hs_ntor/", hs_ntor_tests }, { "hs_ob/", hs_ob_tests }, { "hs_service/", hs_service_tests }, diff --git a/src/test/test.h b/src/test/test.h index ba1d69fa11..56037648d3 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -141,6 +141,7 @@ extern struct testcase_t hs_control_tests[]; extern struct testcase_t hs_descriptor[]; extern struct testcase_t hs_dos_tests[]; extern struct testcase_t hs_intropoint_tests[]; +extern struct testcase_t hs_metrics_tests[]; extern struct testcase_t hs_ntor_tests[]; extern struct testcase_t hs_ob_tests[]; extern struct testcase_t hs_service_tests[]; diff --git a/src/test/test_hs_metrics.c b/src/test/test_hs_metrics.c new file mode 100644 index 0000000000..dd123eb6f1 --- /dev/null +++ b/src/test/test_hs_metrics.c @@ -0,0 +1,68 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_hs_metrics.c + * \brief Test hidden service metrics. + */ + +#define HS_SERVICE_PRIVATE + +#include "test/test.h" +#include "test/test_helpers.h" +#include "test/log_test_helpers.h" + +#include "app/config/config.h" + +#include "feature/hs/hs_metrics.h" +#include "feature/hs/hs_service.h" + +#include "lib/crypt_ops/crypto_ed25519.h" + +static void +test_metrics(void *arg) +{ + hs_service_t *service = NULL; + + (void) arg; + + hs_init(); + + service = hs_service_new(get_options()); + tt_assert(service); + service->config.version = HS_VERSION_THREE; + ed25519_secret_key_generate(&service->keys.identity_sk, 0); + ed25519_public_key_generate(&service->keys.identity_pk, + &service->keys.identity_sk); + register_service(get_hs_service_map(), service); + + tt_assert(service->metrics.store); + + /* Update entry by identifier. */ + hs_metrics_update_by_ident(HS_METRICS_NUM_INTRODUCTIONS, + &service->keys.identity_pk, 0, 42); + + /* Confirm the entry value. */ + const smartlist_t *entries = metrics_store_get_all(service->metrics.store, + "hs_intro_num_total"); + tt_assert(entries); + tt_int_op(smartlist_len(entries), OP_EQ, 1); + const metrics_store_entry_t *entry = smartlist_get(entries, 0); + tt_assert(entry); + tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 42); + + /* Update entry by service now. */ + hs_metrics_update_by_service(HS_METRICS_NUM_INTRODUCTIONS, + service, 0, 42); + tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 84); + + done: + hs_free_all(); +} + +struct testcase_t hs_metrics_tests[] = { + + { "metrics", test_metrics, TT_FORK, NULL, NULL }, + + END_OF_TESTCASES +};

1 0

[tor/master] tests: Fix unit tests after adding new HS metrics module
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit 4c19eed35c8bae22cda219b7a4fb4a65d1c65525 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Oct 22 15:48:05 2020 -0400 tests: Fix unit tests after adding new HS metrics module Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/test/test_hs_service.c | 3 +++ src/test/test_periodic_event.c | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 608e738d85..ad38547495 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -54,6 +54,7 @@ #include "feature/hs/hs_ob.h" #include "feature/hs/hs_cell.h" #include "feature/hs/hs_intropoint.h" +#include "feature/hs/hs_metrics.h" #include "feature/hs/hs_service.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" @@ -664,6 +665,7 @@ test_access_service(void *arg) tt_mem_op(query, OP_EQ, s, sizeof(hs_service_t)); /* Remove service, check if it actually works and then put it back. */ remove_service(global_map, s); + hs_metrics_service_free(s); tt_int_op(get_hs_service_map_size(), OP_EQ, 0); query = find_service(global_map, &s->keys.identity_pk); tt_ptr_op(query, OP_EQ, NULL); @@ -673,6 +675,7 @@ test_access_service(void *arg) tt_int_op(ret, OP_EQ, 0); tt_int_op(get_hs_service_map_size(), OP_EQ, 1); /* Twice should fail. */ + hs_metrics_service_free(s); /* Avoid BUG() on metrics init. */ ret = register_service(global_map, s); tt_int_op(ret, OP_EQ, -1); /* Remove service from map so we don't double free on cleanup. */ diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index b7f1785805..1dd3f7a68d 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -17,6 +17,7 @@ #include "core/or/or.h" #include "app/config/config.h" #include "feature/hibernate/hibernate.h" +#include "feature/hs/hs_metrics.h" #include "feature/hs/hs_service.h" #include "core/mainloop/mainloop.h" #include "core/mainloop/netstatus.h" @@ -187,6 +188,7 @@ test_pe_launch(void *arg) done: if (to_remove) { + hs_metrics_service_free(&service); remove_service(get_hs_service_map(), to_remove); } hs_free_all(); @@ -279,6 +281,7 @@ test_pe_get_roles(void *arg) roles = get_my_roles(options); /* Remove it now so the hs_free_all() doesn't try to free stack memory. */ remove_service(get_hs_service_map(), &service); + hs_metrics_service_free(&service); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY | PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER | @@ -332,6 +335,7 @@ test_pe_hs_service(void *arg) /* Remove the service from the global map, it should trigger a rescan and * disable the HS service events. */ remove_service(get_hs_service_map(), &service); + hs_metrics_service_free(&service); for (int i = 0; mainloop_periodic_events[i].name; ++i) { periodic_event_item_t *item = &mainloop_periodic_events[i]; if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) { @@ -341,6 +345,7 @@ test_pe_hs_service(void *arg) done: if (to_remove) { + hs_metrics_service_free(&service); remove_service(get_hs_service_map(), to_remove); } hs_free_all();

1 0

[tor/master] lib: New metrics library
by ahf＠torproject.org 27 Oct '20

27 Oct '20

commit ec731290a5a790093961f0fdb06cf69000194adf Author: David Goulet <dgoulet(a)torproject.org> Date: Mon Oct 19 15:15:47 2020 -0400 lib: New metrics library Used to provide an interface to create metrics store and update the entries. Related to #40063 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- Makefile.am | 6 +- src/include.am | 3 +- src/lib/metrics/include.am | 25 ++++++ src/lib/metrics/metrics_common.c | 29 +++++++ src/lib/metrics/metrics_common.h | 40 ++++++++++ src/lib/metrics/metrics_store.c | 140 ++++++++++++++++++++++++++++++++++ src/lib/metrics/metrics_store.h | 42 ++++++++++ src/lib/metrics/metrics_store_entry.c | 129 +++++++++++++++++++++++++++++++ src/lib/metrics/metrics_store_entry.h | 68 +++++++++++++++++ src/lib/metrics/prometheus.c | 56 ++++++++++++++ src/lib/metrics/prometheus.h | 18 +++++ 11 files changed, 553 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 50b002139e..96658230f7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -73,7 +73,8 @@ TOR_UTIL_LIBS = \ src/lib/libtor-version.a \ src/lib/libtor-llharden.a \ src/lib/libtor-intmath.a \ - src/lib/libtor-ctime.a + src/lib/libtor-ctime.a \ + src/lib/libtor-metrics.a # Variants of the above for linking the testing variant of tor (for coverage # and tests) @@ -108,7 +109,8 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-version-testing.a \ src/lib/libtor-llharden-testing.a \ src/lib/libtor-intmath.a \ - src/lib/libtor-ctime-testing.a + src/lib/libtor-ctime-testing.a \ + src/lib/libtor-metrics-testing.a endif # Internal crypto libraries used in Tor diff --git a/src/include.am b/src/include.am index 657f6e823a..95acecb057 100644 --- a/src/include.am +++ b/src/include.am @@ -22,10 +22,11 @@ include src/lib/intmath/include.am include src/lib/llharden/include.am include src/lib/lock/include.am include src/lib/log/include.am +include src/lib/malloc/include.am include src/lib/math/include.am include src/lib/memarea/include.am include src/lib/meminfo/include.am -include src/lib/malloc/include.am +include src/lib/metrics/include.am include src/lib/net/include.am include src/lib/osinfo/include.am include src/lib/process/include.am diff --git a/src/lib/metrics/include.am b/src/lib/metrics/include.am new file mode 100644 index 0000000000..62c289446e --- /dev/null +++ b/src/lib/metrics/include.am @@ -0,0 +1,25 @@ + +noinst_LIBRARIES += src/lib/libtor-metrics.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-metrics-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_metrics_a_SOURCES = \ + src/lib/metrics/metrics_store.c \ + src/lib/metrics/metrics_store_entry.c \ + src/lib/metrics/metrics_common.c \ + src/lib/metrics/prometheus.c + +src_lib_libtor_metrics_testing_a_SOURCES = \ + $(src_lib_libtor_metrics_a_SOURCES) +src_lib_libtor_metrics_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_metrics_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/metrics/metrics_store.h \ + src/lib/metrics/metrics_store_entry.h \ + src/lib/metrics/metrics_common.h \ + src/lib/metrics/prometheus.h diff --git a/src/lib/metrics/metrics_common.c b/src/lib/metrics/metrics_common.c new file mode 100644 index 0000000000..5941a4d892 --- /dev/null +++ b/src/lib/metrics/metrics_common.c @@ -0,0 +1,29 @@ +/* 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file metrics_common.c + * @brief Common code for the metrics library + **/ + +#include <stddef.h> + +#include "orconfig.h" + +#include "lib/log/util_bug.h" + +#include "lib/metrics/metrics_common.h" + +/** Return string representation of a metric type. */ +const char * +metrics_type_to_str(const metrics_type_t type) +{ + switch (type) { + case METRICS_TYPE_COUNTER: + return "counter"; + case METRICS_TYPE_GAUGE: + return "gauge"; + default: + tor_assert_unreached(); + } +} diff --git a/src/lib/metrics/metrics_common.h b/src/lib/metrics/metrics_common.h new file mode 100644 index 0000000000..5d1a32ea6c --- /dev/null +++ b/src/lib/metrics/metrics_common.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file metrics_common.h + * @brief Header for lib/metrics/metrics_common.c + **/ + +#ifndef TOR_LIB_METRICS_METRICS_COMMON_H +#define TOR_LIB_METRICS_METRICS_COMMON_H + +#include "lib/cc/torint.h" + +/** Format output type. */ +typedef enum { + /** Prometheus data output format. */ + METRICS_FORMAT_PROMETHEUS = 1, +} metrics_format_t; + +/** Metric type. */ +typedef enum { + /* Increment only. */ + METRICS_TYPE_COUNTER, + /* Can go up or down. */ + METRICS_TYPE_GAUGE, +} metrics_type_t; + +/** Metric counter object (METRICS_TYPE_COUNTER). */ +typedef struct metrics_counter_t { + uint64_t value; +} metrics_counter_t; + +/** Metric gauge object (METRICS_TYPE_GAUGE). */ +typedef struct metrics_gauge_t { + int64_t value; +} metrics_gauge_t; + +const char *metrics_type_to_str(const metrics_type_t type); + +#endif /* !defined(TOR_LIB_METRICS_METRICS_COMMON_H) */ diff --git a/src/lib/metrics/metrics_store.c b/src/lib/metrics/metrics_store.c new file mode 100644 index 0000000000..4f048e103d --- /dev/null +++ b/src/lib/metrics/metrics_store.c @@ -0,0 +1,140 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file metrics_store.c + * @brief Metrics interface to store them based on specific store type and get + * their MetricsPort output. + **/ + +#define METRICS_STORE_ENTRY_PRIVATE + +#include "orconfig.h" + +#include "lib/container/map.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" + +#include "lib/metrics/metrics_store.h" +#include "lib/metrics/metrics_store_entry.h" + +/* Format Drivers. */ +#include "lib/metrics/prometheus.h" + +/** A metric store which contains a map of entries. */ +typedef struct metrics_store_t { + /** Indexed by metrics entry name. An entry is a smartlist_t of one or more + * metrics_store_entry_t allowing for multiple metrics of the same name. + * + * The reason we allow multiple entires is because there are cases where one + * metrics can be used twice by the same entity but with different labels. + * One example is an onion service with multiple ports, the port specific + * metrics will have a port value as a label. */ + strmap_t *entries; +} metrics_store_t; + +/** Function pointer to the format function of a specific driver. */ +typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *); + +/** Helper: Free a single entry in a metrics_store_t taking a void pointer + * parameter. */ +static void +metrics_store_free_void(void *p) +{ + smartlist_t *list = p; + SMARTLIST_FOREACH(list, metrics_store_entry_t *, entry, + metrics_store_entry_free(entry)); + smartlist_free(list); +} + +/** Put the given store output in the buffer data and use the format function + * given in fmt to get it for each entry. */ +static void +get_output(const metrics_store_t *store, buf_t *data, fmt_driver_fn_t fmt) +{ + tor_assert(store); + tor_assert(data); + tor_assert(fmt); + + STRMAP_FOREACH(store->entries, key, const smartlist_t *, entries) { + SMARTLIST_FOREACH_BEGIN(entries, const metrics_store_entry_t *, entry) { + fmt(entry, data); + } SMARTLIST_FOREACH_END(entry); + } STRMAP_FOREACH_END; +} + +/** Return a newly allocated and initialized store of the given type. */ +metrics_store_t * +metrics_store_new(void) +{ + metrics_store_t *store = tor_malloc_zero(sizeof(*store)); + + store->entries = strmap_new(); + + return store; +} + +/** Free the given store including all its entries. */ +void +metrics_store_free_(metrics_store_t *store) +{ + if (store == NULL) { + return; + } + + strmap_free(store->entries, metrics_store_free_void); + tor_free(store); +} + +/** Find all metrics entry in the given store identified by name. If not found, + * NULL is returned. */ +smartlist_t * +metrics_store_get_all(const metrics_store_t *store, const char *name) +{ + tor_assert(store); + tor_assert(name); + + return strmap_get(store->entries, name); +} + +/** Add a new metrics entry to the given store and type. The name MUST be the + * unique identifier. The help string can be omitted. */ +metrics_store_entry_t * +metrics_store_add(metrics_store_t *store, metrics_type_t type, + const char *name, const char *help) +{ + smartlist_t *entries; + metrics_store_entry_t *entry; + + tor_assert(store); + tor_assert(name); + + entries = metrics_store_get_all(store, name); + if (!entries) { + entries = smartlist_new(); + strmap_set(store->entries, name, entries); + } + entry = metrics_store_entry_new(type, name, help); + smartlist_add(entries, entry); + + return entry; +} + +/** Set the output of the given store of the format fmt into the given buffer + * data. */ +void +metrics_store_get_output(const metrics_format_t fmt, + const metrics_store_t *store, buf_t *data) +{ + tor_assert(store); + + switch (fmt) { + case METRICS_FORMAT_PROMETHEUS: + get_output(store, data, prometheus_format_store_entry); + break; + default: + // LCOV_EXCL_START + tor_assert_unreached(); + // LCOV_EXCL_STOP + } +} diff --git a/src/lib/metrics/metrics_store.h b/src/lib/metrics/metrics_store.h new file mode 100644 index 0000000000..9640a5e016 --- /dev/null +++ b/src/lib/metrics/metrics_store.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file metrics_store.h + * @brief Header for lib/metrics/metrics_store.c + **/ + +#ifndef TOR_LIB_METRICS_METRICS_STORE_H +#define TOR_LIB_METRICS_METRICS_STORE_H + +#include "lib/buf/buffers.h" +#include "lib/container/smartlist.h" + +#include "lib/metrics/metrics_common.h" +#include "lib/metrics/metrics_store_entry.h" + +/* Stub. */ +typedef struct metrics_store_t metrics_store_t; + +/* Allocators. */ +void metrics_store_free_(metrics_store_t *store); +#define metrics_store_free(store) \ + FREE_AND_NULL(metrics_store_t, metrics_store_free_, (store)) +metrics_store_t *metrics_store_new(void); + +/* Modifiers. */ +metrics_store_entry_t *metrics_store_add(metrics_store_t *store, + metrics_type_t type, + const char *name, const char *help); + +/* Accessors. */ +smartlist_t *metrics_store_get_all(const metrics_store_t *store, + const char *name); +void metrics_store_get_output(const metrics_format_t fmt, + const metrics_store_t *store, buf_t *data); + +#ifdef METRICS_METRICS_STORE_PRIVATE + +#endif /* METRICS_METRICS_STORE_PRIVATE. */ + +#endif /* !defined(TOR_LIB_METRICS_METRICS_STORE_H) */ diff --git a/src/lib/metrics/metrics_store_entry.c b/src/lib/metrics/metrics_store_entry.c new file mode 100644 index 0000000000..44ebb5cb84 --- /dev/null +++ b/src/lib/metrics/metrics_store_entry.c @@ -0,0 +1,129 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file metrics_store_entry.c + * @brief Metrics store entry which contains the gathered data. + **/ + +#define METRICS_STORE_ENTRY_PRIVATE + +#include <string.h> + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" + +#include "lib/metrics/metrics_store_entry.h" + +/* + * Public API. + */ + +/** Return newly allocated store entry of type COUNTER. */ +metrics_store_entry_t * +metrics_store_entry_new(const metrics_type_t type, const char *name, + const char *help) +{ + metrics_store_entry_t *entry = tor_malloc_zero(sizeof(*entry)); + + tor_assert(name); + + entry->type = type; + entry->name = tor_strdup(name); + entry->labels = smartlist_new(); + if (help) { + entry->help = tor_strdup(help); + } + + return entry; +} + +/** Free a store entry. */ +void +metrics_store_entry_free_(metrics_store_entry_t *entry) +{ + if (!entry) { + return; + } + SMARTLIST_FOREACH(entry->labels, char *, l, tor_free(l)); + smartlist_free(entry->labels); + tor_free(entry->name); + tor_free(entry->help); + tor_free(entry); +} + +/** Update a store entry with value. */ +void +metrics_store_entry_update(metrics_store_entry_t *entry, const int64_t value) +{ + tor_assert(entry); + + switch (entry->type) { + case METRICS_TYPE_COUNTER: + /* Counter can ONLY be positive. */ + if (BUG(value < 0)) { + return; + } + entry->u.counter.value += value; + break; + case METRICS_TYPE_GAUGE: + /* Gauge can increment or decrement. And can be positive or negative. */ + entry->u.gauge.value += value; + break; + } +} + +/** Reset a store entry that is set its metric data to 0. */ +void +metrics_store_entry_reset(metrics_store_entry_t *entry) +{ + tor_assert(entry); + /* Everything back to 0. */ + memset(&entry->u, 0, sizeof(entry->u)); +} + +/** Return store entry value. */ +int64_t +metrics_store_entry_get_value(const metrics_store_entry_t *entry) +{ + tor_assert(entry); + + switch (entry->type) { + case METRICS_TYPE_COUNTER: + if (entry->u.counter.value > INT64_MAX) { + return INT64_MAX; + } + return entry->u.counter.value; + case METRICS_TYPE_GAUGE: + return entry->u.gauge.value; + } + + // LCOV_EXCL_START + tor_assert_unreached(); + // LCOV_EXCL_STOP +} + +/** Add a label into the given entry.*/ +void +metrics_store_entry_add_label(metrics_store_entry_t *entry, + const char *label) +{ + tor_assert(entry); + tor_assert(label); + + smartlist_add(entry->labels, tor_strdup(label)); +} + +/** Return true iff the given entry has the given label. */ +bool +metrics_store_entry_has_label(const metrics_store_entry_t *entry, + const char *label) +{ + tor_assert(entry); + tor_assert(label); + + return smartlist_contains_string(entry->labels, label); +} diff --git a/src/lib/metrics/metrics_store_entry.h b/src/lib/metrics/metrics_store_entry.h new file mode 100644 index 0000000000..6fff9d10eb --- /dev/null +++ b/src/lib/metrics/metrics_store_entry.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file metrics_store_entry.h + * @brief Header for lib/metrics/metrics_store_entry.c + **/ + +#ifndef TOR_LIB_METRICS_METRICS_STORE_ENTRY_H +#define TOR_LIB_METRICS_METRICS_STORE_ENTRY_H + +#include "lib/cc/torint.h" + +#include "lib/metrics/metrics_common.h" + +#ifdef METRICS_STORE_ENTRY_PRIVATE + +/** Metrics store entry. They reside in a metrics_store_t object and are + * opaque to the outside world. */ +typedef struct metrics_store_entry_t { + /** Type of entry. */ + metrics_type_t type; + + /** Name. */ + char *name; + + /** Help comment string. */ + char *help; + + /** Labels attached to that entry. If NULL, no labels. + * + * Labels are used to add extra context to a metrics. For example, a label + * could be an onion address so the metrics can be differentiate. */ + smartlist_t *labels; + + /* Actual data. */ + union { + metrics_counter_t counter; + metrics_gauge_t gauge; + } u; +} metrics_store_entry_t; + +#endif /* METRICS_STORE_ENTRY_PRIVATE */ + +typedef struct metrics_store_entry_t metrics_store_entry_t; + +/* Allocators. */ +metrics_store_entry_t *metrics_store_entry_new(const metrics_type_t type, + const char *name, + const char *help); + +void metrics_store_entry_free_(metrics_store_entry_t *entry); +#define metrics_store_entry_free(entry) \ + FREE_AND_NULL(metrics_store_entry_t, metrics_store_entry_free_, (entry)); + +/* Accessors. */ +int64_t metrics_store_entry_get_value(const metrics_store_entry_t *entry); +bool metrics_store_entry_has_label(const metrics_store_entry_t *entry, + const char *label); + +/* Modifiers. */ +void metrics_store_entry_add_label(metrics_store_entry_t *entry, + const char *label); +void metrics_store_entry_reset(metrics_store_entry_t *entry); +void metrics_store_entry_update(metrics_store_entry_t *entry, + const int64_t value); + +#endif /* !defined(TOR_LIB_METRICS_METRICS_STORE_ENTRY_H) */ diff --git a/src/lib/metrics/prometheus.c b/src/lib/metrics/prometheus.c new file mode 100644 index 0000000000..c2b54e436f --- /dev/null +++ b/src/lib/metrics/prometheus.c @@ -0,0 +1,56 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file prometheus.c + * @brief Metrics format driver for Prometheus data model. + **/ + +#define METRICS_STORE_ENTRY_PRIVATE + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +#include "lib/metrics/prometheus.h" + +/** Return a static buffer containing all the labels properly formatted + * for the output as a string. + * + * Subsequent calls to this invalidates the previous result. */ +static const char * +format_labels(smartlist_t *labels) +{ + static char buf[1024]; + char *line = NULL; + + if (smartlist_len(labels) == 0) { + buf[0] = '\0'; + goto end; + } + + line = smartlist_join_strings(labels, ",", 0, NULL); + tor_snprintf(buf, sizeof(buf), "{%s}", line); + + end: + tor_free(line); + return buf; +} + +/** Format the given entry in to the buffer data. */ +void +prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data) +{ + tor_assert(entry); + tor_assert(data); + + buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); + buf_add_printf(data, "# TYPE %s %s\n", entry->name, + metrics_type_to_str(entry->type)); + buf_add_printf(data, "%s%s %" PRIi64 "\n", entry->name, + format_labels(entry->labels), + metrics_store_entry_get_value(entry)); +} diff --git a/src/lib/metrics/prometheus.h b/src/lib/metrics/prometheus.h new file mode 100644 index 0000000000..eea26e8ac4 --- /dev/null +++ b/src/lib/metrics/prometheus.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file prometheus.h + * @brief Header for feature/metrics/prometheus.c + **/ + +#ifndef TOR_LIB_METRICS_PROMETHEUS_H +#define TOR_LIB_METRICS_PROMETHEUS_H + +#include "lib/buf/buffers.h" +#include "lib/metrics/metrics_store_entry.h" + +void prometheus_format_store_entry(const metrics_store_entry_t *entry, + buf_t *data); + +#endif /* !defined(TOR_LIB_METRICS_PROMETHEUS_H) */

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits October 2020