February 2020 - tor-commits - lists.torproject.org

[gettor/master] Add tests for database functions
by cohosh＠torproject.org 13 Feb '20

13 Feb '20

commit 3d6fb34e712f97d646b7502e62f6b6aba8a5382c Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Mon Feb 10 09:52:48 2020 -0500 Add tests for database functions --- tests/test_db.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..d663d89 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import pytest +from datetime import datetime +from twisted.trial import unittest + +from . import conftests + +class DatabaseTests(unittest.TestCase): + + # Fail any tests which take longer than 15 seconds. + timeout = 15 + def setUp(self): + self.settings = conftests.options.parse_settings("en","tests/test.conf.json") + print(self.settings.get("dbname")) + self.db = conftests.SQLite3(self.settings.get("dbname")) + + def tearDown(self): + print("tearDown()") + + def add_dummy_requests(self, num): + now_str = datetime.now().strftime("%Y%m%d") + for i in (0, num): + self.db.new_request( + id='testid', + command='links', + platform='linux', + language='en', + service='email', + date=now_str, + status="ONHOLD", + ) + + def test_stored_locales(self): + locales = self.db.get_locales() + self.assertIn('en-US', locales) + + def test_requests(self): + now_str = datetime.now().strftime("%Y%m%d") + self.add_dummy_requests(2) + num = self.db.get_num_requests("testid", "email") + self.assertEqual(num, 2) + + requests = self.db.get_requests("ONHOLD", "links", "email") + for request in requests: + print(request) + self.assertEqual(request[1], "links") + self.assertEqual(request[4], "email") + self.assertEqual(request[5], now_str) + self.assertEqual(request[6], "ONHOLD") + self.assertEqual(len(requests), 2) + + self.db.remove_request("testid", "email", now_str) + num = self.db.get_num_requests("testid", "email") + self.assertEqual(num, 0) + + def test_links(self): + links = self.db.get_links("linux", "en-US", "ACTIVE") + self.assertEqual(len(links), 2) # Right now we have github and gitlab + + for link in links: + self.assertEqual(link[1], "linux") + self.assertEqual(link[2], "en-US") + self.assertEqual(link[6], "ACTIVE") + self.assertIn(link[5], ["github", "gitlab"]) + +if __name__ == "__main__": + unittest.main()

1 0

[gettor/master] Switch from twisted adbapi to sqlite3 package
by cohosh＠torproject.org 13 Feb '20

13 Feb '20

commit e4f6d98fb9d0c82d154c12cbb5edaa2daf2572dd Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Fri Feb 7 19:09:47 2020 -0500 Switch from twisted adbapi to sqlite3 package The rest of GetTor uses the python sqlite3 package, and it's much simpler to use and to test. --- README.md | 2 ++ gettor/utils/db.py | 86 ++++++++++++++++++++++++++---------------------------- tests/conftests.py | 1 + 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index f31d113..6ad9603 100644 --- a/README.md +++ b/README.md @@ -82,5 +82,7 @@ GetTor includes PyTest unit tests. To run the tests, first install the dependenc ``` +$ python3 scripts/create_db -n -c -o -f tests/gettor.db +$ python3 scripts/add_links_to_db -f tests/gettor.db $ pytest-3 tests/ ``` diff --git a/gettor/utils/db.py b/gettor/utils/db.py index 0ca11aa..7c3853f 100644 --- a/gettor/utils/db.py +++ b/gettor/utils/db.py @@ -9,10 +9,10 @@ from __future__ import absolute_import +import sqlite3 from datetime import datetime from twisted.python import log -from twisted.enterprise import adbapi class SQLite3(object): """ @@ -20,94 +20,90 @@ class SQLite3(object): """ def __init__(self, dbname): """Constructor.""" - self.dbpool = adbapi.ConnectionPool( - "sqlite3", dbname, check_same_thread=False - ) - - def query_callback(self, results=None): - """ - Query callback - Log that the database query has been executed and return results - """ - log.msg("Database query executed successfully.") - return results - - def query_errback(self, error=None): - """ - Query error callback - Logs database error - """ - if error: - log.msg("Database error: {}".format(error)) - return None + self.conn = sqlite3.connect(dbname) def new_request(self, id, command, service, platform, language, date, status): """ Perform a new request to the database """ + c = self.conn.cursor() query = "INSERT INTO requests VALUES(?, ?, ?, ?, ?, ?, ?)" - return self.dbpool.runQuery( - query, (id, command, platform, language, service, date, status) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (id, command, platform, language, service, + date, status)) + self.conn.commit() + return def get_requests(self, status, command, service): """ Perform a SELECT request to the database """ + c = self.conn.cursor() query = "SELECT * FROM requests WHERE service=? AND command=? AND "\ "status = ?" - return self.dbpool.runQuery( - query, (service, command, status) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (service, command, status)) + + return c.fetchall() def get_num_requests(self, id, service): """ Get number of requests for statistics """ - query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND service=?" + c = self.conn.cursor() + query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND "\ + "service=?" - return self.dbpool.runQuery( - query, (id, service) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (id, service)) + return c.fetchone()[0] def remove_request(self, id, service, date): """ Removes completed request record from the database """ - query = "DELETE FROM requests WHERE id=? AND service=? AND date=?" + c = self.conn.cursor() + query = "DELETE FROM requests WHERE id=? AND service=? AND "\ + "date=?" - return self.dbpool.runQuery( - query, (id, service, date) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (id, service, date)) + self.conn.commit() + return def update_stats(self, command, service, platform=None, language='en'): """ Update statistics to the database """ + c = self.conn.cursor() now_str = datetime.now().strftime("%Y%m%d") query = "INSERT INTO stats(num_requests, platform, language, command, "\ - "service, date) VALUES (1, ?, ?, ?, ?, ?) ON CONFLICT(platform, "\ - "language, command, service, date) DO UPDATE SET num_requests=num_requests+1" + "service, date) VALUES (1, ?, ?, ?, ?, ?) ON "\ + "CONFLICT(platform, language, command, service, date) "\ + "DO UPDATE SET num_requests=num_requests+1" - return self.dbpool.runQuery( - query, (platform, language, command, service, now_str) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (platform, language, command, service, + now_str)) + self.conn.commit() + return def get_links(self, platform, language, status): """ Get links from the database per platform """ + c = self.conn.cursor() query = "SELECT * FROM links WHERE platform=? AND language=? AND status=?" - return self.dbpool.runQuery( - query, (platform, language, status) - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query, (platform, language, status)) + + return c.fetchall() def get_locales(self): """ Get a list of the supported tor browser binary locales """ + c = self.conn.cursor() query = "SELECT DISTINCT language FROM links" - return self.dbpool.runQuery(query - ).addCallback(self.query_callback).addErrback(self.query_errback) + c.execute(query) + + locales = [] + for locale in c.fetchall(): + locales.append(locale[0]) + return locales diff --git a/tests/conftests.py b/tests/conftests.py index cbb4d28..d509776 100644 --- a/tests/conftests.py +++ b/tests/conftests.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals from gettor.utils import options from gettor.utils import strings from gettor.utils import twitter +from gettor.utils.db import SQLite3 from gettor.services.email.sendmail import Sendmail from gettor.services.twitter import twitterdm from gettor.parse.email import EmailParser, AddressError, DKIMError

1 0

[metrics-cloud/master] cfn/onionperf: Add DNS RR creation for instance
by irl＠torproject.org 13 Feb '20

13 Feb '20

commit c45a60fcf8f6140f863a6694173e92b9a2127440 Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Oct 31 12:23:10 2019 +0000 cfn/onionperf: Add DNS RR creation for instance --- cloudformation/onionperf-dev.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cloudformation/onionperf-dev.yml b/cloudformation/onionperf-dev.yml index 6ac711c..474b8c2 100644 --- a/cloudformation/onionperf-dev.yml +++ b/cloudformation/onionperf-dev.yml @@ -23,6 +23,15 @@ Resources: - Fn::ImportValue: !Sub 'MetricsPingableSecurityGroup' - Fn::ImportValue: !Sub 'MetricsHTTPSecurityGroup' - Fn::ImportValue: !Sub 'MetricsHTTPSSecurityGroup' + OnionperfDNS: + Type: AWS::Route53::RecordSet + Properties: + HostedZoneName: tm-dev-aws.safemetrics.org. + Name: !Join ['', [!Ref 'AWS::StackName', .tm-dev-aws.safemetrics.org.]] + Type: A + TTL: '300' + ResourceRecords: + - !GetAtt Instance.PublicIp Outputs: PublicIp: Description: "Instance public IP"

1 0

[tor/maint-0.4.3] Merge branch 'ticket33290_v2_043' into maint-0.4.3
by nickm＠torproject.org 13 Feb '20

13 Feb '20

commit 6c5307e42f74d4f0b0a0541fa7e8fe3cb246403a Merge: d0c335021 baeff46d3 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 13 10:47:32 2020 -0500 Merge branch 'ticket33290_v2_043' into maint-0.4.3 changes/ticket33290 | 4 ++++ src/core/or/circuitmux.c | 8 ++++++-- src/core/or/circuitmux_ewma.c | 4 +++- src/core/or/circuitmux_ewma.h | 9 ++++++--- 4 files changed, 19 insertions(+), 6 deletions(-)

1 0

[tor/master] Before freeing ewma objects, use memwipe instead of resetting magic.
by nickm＠torproject.org 13 Feb '20

13 Feb '20

commit fff1054d17c8e869c38d4ca1c68ed74a7e82e393 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 13 09:11:35 2020 -0500 Before freeing ewma objects, use memwipe instead of resetting magic. --- src/core/or/circuitmux_ewma.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c index 606b755e2..07f83af0f 100644 --- a/src/core/or/circuitmux_ewma.c +++ b/src/core/or/circuitmux_ewma.c @@ -38,6 +38,7 @@ #include "core/or/circuitmux.h" #include "core/or/circuitmux_ewma.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_util.h" #include "feature/nodelist/networkstatus.h" #include "app/config/or_options_st.h" @@ -299,7 +300,7 @@ ewma_free_cmux_data(circuitmux_t *cmux, pol = TO_EWMA_POL_DATA(pol_data); smartlist_free(pol->active_circuit_pqueue); - pol->base_.magic = 0xDEAD901C; + memwipe(pol, 0xda, sizeof(ewma_policy_data_t)); tor_free(pol); } @@ -366,7 +367,7 @@ ewma_free_circ_data(circuitmux_t *cmux, if (!pol_circ_data) return; cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data); - cdata->base_.magic = 0xDEADC14C; + memwipe(cdata, 0xdc, sizeof(ewma_policy_circ_data_t)); tor_free(cdata); }

1 0

[tor/master] Use more memory poisoning and better asserts around ewma code
by nickm＠torproject.org 13 Feb '20

13 Feb '20

commit 80e3dc47272c9ba423d40ce367fb99d39c3150ec Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Feb 12 14:17:19 2020 -0500 Use more memory poisoning and better asserts around ewma code Attempt to diagnose 32464; fixes 33290. --- changes/ticket33290 | 4 ++++ src/core/or/circuitmux.c | 8 ++++++-- src/core/or/circuitmux_ewma.c | 11 ++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/changes/ticket33290 b/changes/ticket33290 new file mode 100644 index 000000000..882764020 --- /dev/null +++ b/changes/ticket33290 @@ -0,0 +1,4 @@ + o Minor features (diagnostic): + - Improve assertions and add some memory-poisoning code to try to track + down possible causes of a rare crash (32564) in the EWMA code. + Closes ticket 33290. diff --git a/src/core/or/circuitmux.c b/src/core/or/circuitmux.c index b2628bec3..72f6ba662 100644 --- a/src/core/or/circuitmux.c +++ b/src/core/or/circuitmux.c @@ -79,6 +79,8 @@ #include "core/or/destroy_cell_queue_st.h" #include "core/or/or_circuit_st.h" +#include "lib/crypt_ops/crypto_util.h" + /* * Private typedefs for circuitmux.c */ @@ -973,7 +975,10 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) /* Now remove it from the map */ HT_REMOVE(chanid_circid_muxinfo_map, cmux->chanid_circid_map, hashent); - /* Free the hash entry */ + /* Wipe and free the hash entry */ + // This isn't sensitive, but we want to be sure to know if we're accessing + // this accidentally. + memwipe(hashent, 0xef, sizeof(hashent)); tor_free(hashent); } } @@ -1334,4 +1339,3 @@ circuitmux_compare_muxes, (circuitmux_t *cmux_1, circuitmux_t *cmux_2)) return 0; } } - diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c index 3f83c3fd5..606b755e2 100644 --- a/src/core/or/circuitmux_ewma.c +++ b/src/core/or/circuitmux_ewma.c @@ -147,7 +147,9 @@ TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol) { if (!pol) return NULL; else { - tor_assert(pol->magic == EWMA_POL_DATA_MAGIC); + tor_assertf(pol->magic == EWMA_POL_DATA_MAGIC, + "Mismatch: %"PRIu32" != %"PRIu32, + pol->magic, EWMA_POL_DATA_MAGIC); return DOWNCAST(ewma_policy_data_t, pol); } } @@ -162,7 +164,9 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol) { if (!pol) return NULL; else { - tor_assert(pol->magic == EWMA_POL_CIRC_DATA_MAGIC); + tor_assertf(pol->magic == EWMA_POL_CIRC_DATA_MAGIC, + "Mismatch: %"PRIu32" != %"PRIu32, + pol->magic, EWMA_POL_CIRC_DATA_MAGIC); return DOWNCAST(ewma_policy_circ_data_t, pol); } } @@ -295,6 +299,7 @@ ewma_free_cmux_data(circuitmux_t *cmux, pol = TO_EWMA_POL_DATA(pol_data); smartlist_free(pol->active_circuit_pqueue); + pol->base_.magic = 0xDEAD901C; tor_free(pol); } @@ -361,7 +366,7 @@ ewma_free_circ_data(circuitmux_t *cmux, if (!pol_circ_data) return; cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data); - + cdata->base_.magic = 0xDEADC14C; tor_free(cdata); }

1 0

[tor/master] Merge branch 'ticket33290_v2_043' into maint-0.4.3
by nickm＠torproject.org 13 Feb '20

13 Feb '20

commit 6c5307e42f74d4f0b0a0541fa7e8fe3cb246403a Merge: d0c335021 baeff46d3 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 13 10:47:32 2020 -0500 Merge branch 'ticket33290_v2_043' into maint-0.4.3 changes/ticket33290 | 4 ++++ src/core/or/circuitmux.c | 8 ++++++-- src/core/or/circuitmux_ewma.c | 4 +++- src/core/or/circuitmux_ewma.h | 9 ++++++--- 4 files changed, 19 insertions(+), 6 deletions(-)

1 0

[tor/master] Merge branch 'ticket33290_v2_042' into ticket33290_v2_043
by nickm＠torproject.org 13 Feb '20

13 Feb '20

commit baeff46d3625002a9de5781cd86b6c45834bbe56 Merge: d0c335021 fd1686c7d Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 13 09:21:47 2020 -0500 Merge branch 'ticket33290_v2_042' into ticket33290_v2_043 Conflicts: src/core/or/circuitmux_ewma.c changes/ticket33290 | 4 ++++ src/core/or/circuitmux.c | 8 ++++++-- src/core/or/circuitmux_ewma.c | 4 +++- src/core/or/circuitmux_ewma.h | 9 ++++++--- 4 files changed, 19 insertions(+), 6 deletions(-) diff --cc src/core/or/circuitmux.c index 0e932f032,72f6ba662..1f79ea4d9 --- a/src/core/or/circuitmux.c +++ b/src/core/or/circuitmux.c @@@ -77,8 -75,12 +77,10 @@@ #include "core/or/circuitmux.h" #include "core/or/relay.h" -#include "core/or/cell_queue_st.h" -#include "core/or/destroy_cell_queue_st.h" #include "core/or/or_circuit_st.h" + #include "lib/crypt_ops/crypto_util.h" + /* * Private typedefs for circuitmux.c */ diff --cc src/core/or/circuitmux_ewma.h index fc7e7ebf7,b45ce1f91..e41cf9e0f --- a/src/core/or/circuitmux_ewma.h +++ b/src/core/or/circuitmux_ewma.h @@@ -22,114 -22,9 +22,117 @@@ void cmux_ewma_set_options(const or_opt void circuitmux_ewma_free_all(void); #ifdef CIRCUITMUX_EWMA_PRIVATE + +/*** EWMA structures ***/ + +typedef struct cell_ewma_t cell_ewma_t; +typedef struct ewma_policy_data_t ewma_policy_data_t; +typedef struct ewma_policy_circ_data_t ewma_policy_circ_data_t; + +/** + * The cell_ewma_t structure keeps track of how many cells a circuit has + * transferred recently. It keeps an EWMA (exponentially weighted moving + * average) of the number of cells flushed from the circuit queue onto a + * connection in channel_flush_from_first_active_circuit(). + */ + +struct cell_ewma_t { + /** The last 'tick' at which we recalibrated cell_count. + * + * A cell sent at exactly the start of this tick has weight 1.0. Cells sent + * since the start of this tick have weight greater than 1.0; ones sent + * earlier have less weight. */ + unsigned int last_adjusted_tick; + /** The EWMA of the cell count. */ + double cell_count; + /** True iff this is the cell count for a circuit's previous + * channel. */ + unsigned int is_for_p_chan : 1; + /** The position of the circuit within the OR connection's priority + * queue. */ + int heap_index; +}; + +struct ewma_policy_data_t { + circuitmux_policy_data_t base_; + + /** + * Priority queue of cell_ewma_t for circuits with queued cells waiting + * for room to free up on the channel that owns this circuitmux. Kept + * in heap order according to EWMA. This was formerly in channel_t, and + * in or_connection_t before that. + */ + smartlist_t *active_circuit_pqueue; + + /** + * The tick on which the cell_ewma_ts in active_circuit_pqueue last had + * their ewma values rescaled. This was formerly in channel_t, and in + * or_connection_t before that. + */ + unsigned int active_circuit_pqueue_last_recalibrated; +}; + +struct ewma_policy_circ_data_t { + circuitmux_policy_circ_data_t base_; + + /** + * The EWMA count for the number of cells flushed from this circuit + * onto this circuitmux. Used to determine which circuit to flush + * from next. This was formerly in circuit_t and or_circuit_t. + */ + cell_ewma_t cell_ewma; + + /** + * Pointer back to the circuit_t this is for; since we're separating + * out circuit selection policy like this, we can't attach cell_ewma_t + * to the circuit_t any more, so we can't use SUBTYPE_P directly to a + * circuit_t like before; instead get it here. + */ + circuit_t *circ; +}; + +#define EWMA_POL_DATA_MAGIC 0x2fd8b16aU +#define EWMA_POL_CIRC_DATA_MAGIC 0x761e7747U + +/*** Downcasts for the above types ***/ + +/** + * Downcast a circuitmux_policy_data_t to an ewma_policy_data_t and assert + * if the cast is impossible. + */ + +static inline ewma_policy_data_t * +TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol) +{ + if (!pol) return NULL; + else { - tor_assert(pol->magic == EWMA_POL_DATA_MAGIC); ++ tor_assertf(pol->magic == EWMA_POL_DATA_MAGIC, ++ "Mismatch: %"PRIu32" != %"PRIu32, ++ pol->magic, EWMA_POL_DATA_MAGIC); + return DOWNCAST(ewma_policy_data_t, pol); + } +} + +/** + * Downcast a circuitmux_policy_circ_data_t to an ewma_policy_circ_data_t + * and assert if the cast is impossible. + */ + +static inline ewma_policy_circ_data_t * +TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol) +{ + if (!pol) return NULL; + else { - tor_assert(pol->magic == EWMA_POL_CIRC_DATA_MAGIC); ++ tor_assertf(pol->magic == EWMA_POL_CIRC_DATA_MAGIC, ++ "Mismatch: %"PRIu32" != %"PRIu32, ++ pol->magic, EWMA_POL_CIRC_DATA_MAGIC); + return DOWNCAST(ewma_policy_circ_data_t, pol); + } +} + STATIC unsigned cell_ewma_get_current_tick_and_fraction(double *remainder_out); STATIC void cell_ewma_initialize_ticks(void); -#endif -#endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */ +#endif /* defined(CIRCUITMUX_EWMA_PRIVATE) */ +#endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */ -

1 0

[tor/master] Merge branch 'ticket33290_v2_041' into ticket33290_v2_042
by nickm＠torproject.org 13 Feb '20

13 Feb '20

commit fd1686c7d8df8d8159fcaab1c59dfabe154afd88 Merge: ff5220536 fff1054d1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 13 09:12:48 2020 -0500 Merge branch 'ticket33290_v2_041' into ticket33290_v2_042 changes/ticket33290 | 4 ++++ src/core/or/circuitmux.c | 8 ++++++-- src/core/or/circuitmux_ewma.c | 12 +++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-)

1 0

[tor/release-0.4.3] Merge branch 'ticket33290_v2_041' into ticket33290_v2_042
by nickm＠torproject.org 13 Feb '20

13 Feb '20

commit fd1686c7d8df8d8159fcaab1c59dfabe154afd88 Merge: ff5220536 fff1054d1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 13 09:12:48 2020 -0500 Merge branch 'ticket33290_v2_041' into ticket33290_v2_042 changes/ticket33290 | 4 ++++ src/core/or/circuitmux.c | 8 ++++++-- src/core/or/circuitmux_ewma.c | 12 +++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-)

1 0