July 2020 - tor-commits - lists.torproject.org

[bridgedb/master] Add favicon to BridgeDB's web UI.
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit 20f104955c6b798c7452d7ee09ae45fbbc99c62d Author: Philipp Winter <phw(a)nymity.ch> Date: Thu May 28 13:50:02 2020 -0700 Add favicon to BridgeDB's web UI. This fixes <https://bugs.torproject.org/19774>. --- CHANGELOG | 3 +++ .../https/templates/assets/images/favicon.ico | Bin 0 -> 4286 bytes bridgedb/distributors/https/templates/base.html | 1 + setup.py | 3 ++- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 6dfa43c..754e2f0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ + * FIXES https://bugs.torproject.org/19774 + Add a favicon to BridgeDB's web UI. + Changes in version 0.10.1 - 2020-05-27 * FIXES https://bugs.torproject.org/33945 diff --git a/bridgedb/distributors/https/templates/assets/images/favicon.ico b/bridgedb/distributors/https/templates/assets/images/favicon.ico new file mode 100644 index 0000000..2f24bf9 Binary files /dev/null and b/bridgedb/distributors/https/templates/assets/images/favicon.ico differ diff --git a/bridgedb/distributors/https/templates/base.html b/bridgedb/distributors/https/templates/base.html index 2f502a9..1f16ca0 100644 --- a/bridgedb/distributors/https/templates/base.html +++ b/bridgedb/distributors/https/templates/base.html @@ -13,6 +13,7 @@ <meta name="description" content="Tor Bridges"> <meta name="author" content="The Tor Project"> + <link rel="icon" type="image/x-icon" href="/assets/images/favicon.ico"> <link rel="stylesheet" href="/assets/css/bootstrap.min.css"> <link rel="stylesheet" href="/assets/css/font-awesome.min.css"> <link rel="stylesheet" href="/assets/css/main.css"> diff --git a/setup.py b/setup.py index 18c391b..74b6254 100644 --- a/setup.py +++ b/setup.py @@ -175,7 +175,8 @@ def get_template_files(): 'assets/font/*.svg', 'assets/font/*.eot', 'assets/js/*.js', - 'assets/images/*.svg'] + 'assets/images/*.svg', + 'assets/images/*.ico'] template_files = [] for include_pattern in include_patterns:

1 0

[bridgedb/master] Avoid giving out bridges with broken tor versions.
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit 8d169fc45c8cfd7c8c9028d125bff4dbb3afafc6 Author: Philipp Winter <phw(a)nymity.ch> Date: Fri Jun 12 10:05:26 2020 -0700 Avoid giving out bridges with broken tor versions. This patch makes BridgeDB avoid giving out bridges that are affected by the following bug: <https://bugs.torproject.org/28912> This fixes <https://bugs.torproject.org/29184>. --- CHANGELOG | 5 +++++ bridgedb.conf | 10 ++++++++++ bridgedb/bridges.py | 15 +++++++++++++++ bridgedb/main.py | 7 +++++++ bridgedb/parse/versions.py | 29 +++++++++++++++++++++++++++++ bridgedb/test/test_bridges.py | 27 +++++++++++++++++++++++++++ 6 files changed, 93 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 754e2f0..c712436 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ + * FIXES https://bugs.torproject.org/29184 + Add a new configuration option, BLACKLISTED_TOR_VERSIONS, which contains a + list of Tor versions. BridgeDB won't hand out bridges whose Tor version + is present in this blacklist. + * FIXES https://bugs.torproject.org/19774 Add a favicon to BridgeDB's web UI. diff --git a/bridgedb.conf b/bridgedb.conf index 31eb3fa..6d4839d 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -309,6 +309,16 @@ DEFAULT_TRANSPORT = 'obfs4' # Accept-Language,[Kk]lingon BLACKLISTED_REQUEST_HEADERS_FILE="blacklisted-request-headers.csv" +# List of tuples that specify blacklisted tor version ranges. The first +# element marks the start of the range and the second element marks the end. +# Both the start *and* the end version are blocked too. If you want to block a +# single version, have the start and end range be identical. BridgeDB won't +# distribute bridges whose version falls within any version ranges. +BLACKLISTED_TOR_VERSIONS = [ + ('0.3.4', '0.3.4.9'), # See <https://bugs.torproject.org/29184>. + ('0.3.5', '0.3.5.6') +] + # Decoy bridges that we are handing out to bots that we detected using the # regular expressions in BLACKLISTED_REQUEST_HEADERS_FILE. The CSV file must # have the following format: diff --git a/bridgedb/bridges.py b/bridgedb/bridges.py index cd01948..6ec864d 100644 --- a/bridgedb/bridges.py +++ b/bridgedb/bridges.py @@ -1825,3 +1825,18 @@ class Bridge(BridgeBackwardsCompatibility): logging.info("Removing dead transport for bridge %s: %s %s:%s %s" % (self, pt.methodname, pt.address, pt.port, pt.arguments)) self.transports.remove(pt) + + def runsVersion(self, version_tuples): + """Return ``True`` if this bridge runs any of the given versions. + + :param list version_tuples: A list of tuples that contain a minimum and + maximum version number (as :class:`stem.version.Version` objects), + each. + :rtype: bool + :returns: ``True`` if this bridge runs any of the given Tor versions + and ``False`` otherwise. + """ + for min_version, max_version in version_tuples: + if min_version <= self.software <= max_version: + return True + return False diff --git a/bridgedb/main.py b/bridgedb/main.py index 70001a9..44d0668 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -38,6 +38,7 @@ from bridgedb.distributors.https.distributor import HTTPSDistributor from bridgedb.distributors.moat.distributor import MoatDistributor from bridgedb.parse import descriptors from bridgedb.parse.blacklist import parseBridgeBlacklistFile +from bridgedb.parse.versions import parseVersionsList import bridgedb.Storage @@ -211,6 +212,10 @@ def load(state, hashring, clear=False): elif bridge in blacklist.keys(): logging.warn("Not distributing blacklisted Bridge %s %s:%s: %s" % (bridge, bridge.address, bridge.orPort, blacklist[bridge])) + # Skip bridges that are running a blacklisted version of Tor. + elif bridge.runsVersion(state.BLACKLISTED_TOR_VERSIONS): + logging.warn("Not distributing bridge %s because it runs blacklisted " + "Tor version %s." % (router.fingerprint, bridge.software)) else: # If the bridge is not running, then it is skipped during the # insertion process. @@ -418,6 +423,8 @@ def run(options, reactor=reactor): proxy.loadProxiesFromFile(proxyfile, proxies, removeStale=True) metrics.setProxies(proxies) + state.BLACKLISTED_TOR_VERSIONS = parseVersionsList(state.BLACKLISTED_TOR_VERSIONS) + logging.info("Reloading blacklisted request headers...") antibot.loadBlacklistedRequestHeaders(config.BLACKLISTED_REQUEST_HEADERS_FILE) logging.info("Reloading decoy bridges...") diff --git a/bridgedb/parse/versions.py b/bridgedb/parse/versions.py index 335e04c..832969e 100644 --- a/bridgedb/parse/versions.py +++ b/bridgedb/parse/versions.py @@ -23,6 +23,10 @@ bridgedb.parse.versions .. """ +import logging + +import stem.version + from twisted import version as _txversion # The twisted.python.util.Version class was moved in Twisted==14.0.0 to @@ -127,3 +131,28 @@ class Version(_Version): str(self.minor), str(self.micro), str(prerelease)) + + +def parseVersionsList(versions_list): + """Turn the given version strings into stem objects. + + :param list versions_list: A list of tuples. Each tuple contains a minimum + and maximum version number as strings. + :rtype: list + :returns: A list of tuples. Each tuple contains a minimum and maximum + version number as :class:`stem.version.Version` objects. + """ + parsed = [] + for v1, v2 in versions_list: + # We're dealing with an already-parsed version list. + if isinstance(v1, stem.version.Version): + return versions_list + try: + parsed.append(tuple([stem.version.Version(v1), + stem.version.Version(v2)])) + except ValueError: + logging.error("Couldn't parse BLACKLISTED_TOR_VERSIONS; " + "probably because of badly formatted " + "configuration file. Ignoring config option.") + return [] + return parsed diff --git a/bridgedb/test/test_bridges.py b/bridgedb/test/test_bridges.py index fd14e87..82d9584 100644 --- a/bridgedb/test/test_bridges.py +++ b/bridgedb/test/test_bridges.py @@ -19,6 +19,8 @@ import hashlib import os import warnings +import stem.version + from twisted.trial import unittest from bridgedb import bridges @@ -1837,3 +1839,28 @@ class BridgeTests(unittest.TestCase): self.assertTrue(len(self.bridge.transports), 3) self.assertNotIn('scramblesuit', [pt.methodname for pt in self.bridge.transports]) + + def test_runsVersions(self): + """Calling runsVersions() should tell us if a bridge is running any of + the given versions. + """ + self.bridge.software = stem.version.Version("0.1.2.3") + + t1 = tuple([stem.version.Version("0.1.2.3"), + stem.version.Version("0.1.2.3")]) + self.assertTrue(self.bridge.runsVersion([t1])) + + t2 = tuple([stem.version.Version("0.1.2"), + stem.version.Version("0.1.2")]) + self.assertFalse(self.bridge.runsVersion([t2])) + + t3 = tuple([stem.version.Version("0.1.2"), + stem.version.Version("0.1.2.5")]) + self.assertTrue(self.bridge.runsVersion([t3])) + + t4 = tuple([stem.version.Version("0.2.2"), + stem.version.Version("0.3.1")]) + self.assertFalse(self.bridge.runsVersion([t4])) + + self.assertTrue(self.bridge.runsVersion([t1, t2, t3, t4])) + self.assertFalse(self.bridge.runsVersion([t2, t4]))

1 0

[bridgedb/master] Merge branch 'enhancement/29184' into 'develop'
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit 28c5479c455a499e4ac77203920dc8cc62981d29 Merge: 6d3e899 8d169fc Author: Philipp Winter <phw(a)torproject.org> Date: Tue Jun 23 23:08:36 2020 +0000 Merge branch 'enhancement/29184' into 'develop' Avoid giving out bridges with broken tor versions. See merge request tpo/anti-censorship/bridgedb!3 CHANGELOG | 5 +++++ bridgedb.conf | 10 ++++++++++ bridgedb/bridges.py | 15 +++++++++++++++ bridgedb/main.py | 7 +++++++ bridgedb/parse/versions.py | 29 +++++++++++++++++++++++++++++ bridgedb/test/test_bridges.py | 27 +++++++++++++++++++++++++++ 6 files changed, 93 insertions(+)

1 0

[bridgedb/master] Remove the --reload command line switch.
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit 2351f8652bb99b908c36957edee1585319ff6fcb Author: Philipp Winter <phw(a)nymity.ch> Date: Thu Jul 2 10:53:28 2020 -0700 Remove the --reload command line switch. This fixes tpo/anti-censorship/bridgedb#40001. --- CHANGELOG | 3 +++ README.rst | 4 ---- bridgedb/parse/options.py | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c712436..0351de7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ + * FIXES https://gitlab.torproject.org/tpo/anti-censorship/bridgedb/-/issues/40001 + Remove the --reload command line switch. It doesn't actually do anything. + * FIXES https://bugs.torproject.org/29184 Add a new configuration option, BLACKLISTED_TOR_VERSIONS, which contains a list of Tor versions. BridgeDB won't hand out bridges whose Tor version diff --git a/README.rst b/README.rst index 1b0a840..51eff40 100644 --- a/README.rst +++ b/README.rst @@ -318,10 +318,6 @@ Reloading Bridges From Their Descriptor Files: When you have new lists of bridges from the Bridge Authority, replace the old files and do:: - bridgedb --reload - -Or just give it a SIGHUP:: - kill -s SIGHUP `cat .../run/bridgedb.pid` diff --git a/bridgedb/parse/options.py b/bridgedb/parse/options.py index c32ada3..83c0984 100644 --- a/bridgedb/parse/options.py +++ b/bridgedb/parse/options.py @@ -290,8 +290,6 @@ class SIGUSR1Options(BaseOptions): class MainOptions(BaseOptions): """Main commandline options parser for BridgeDB.""" - optFlags = [ - ['reload', 'R', 'Reload bridge descriptors into running servers']] subCommands = [ ['mock', None, MockOptions, "Generate a testing environment"], ['SIGHUP', None, SIGHUPOptions,

1 0

[bridgedb/master] Make BridgeDB export internal metrics.
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit ce32921680158e8d977f72c16f05de1f9f69cc85 Author: Philipp Winter <phw(a)nymity.ch> Date: Mon Dec 2 15:23:25 2019 -0800 Make BridgeDB export internal metrics. BridgeDB already exports usage metrics that help us understand how our users interact with BridgeDB. This commit adds internal metrics, which shed light on BridgeDB's internals. In particular, we add the following internal metrics: * Number of bridges per distributor subring. * Number of empty responses per distributor. * Number of IPv4/IPv6 handouts. * Descriptive statistics on the number of bridge handouts (min, max, median, quartile 1, quartile 3, upper whisker, lower whisker). * Number of unique bridges that were handed out. This patch fixes tpo/anti-censorship/bridgedb#31422 and also bumps BridgeDB's metrics format to version 2. --- CHANGELOG | 4 + bridgedb/distributors/email/autoresponder.py | 12 +- bridgedb/distributors/https/server.py | 22 ++- bridgedb/distributors/moat/server.py | 47 ++++-- bridgedb/main.py | 17 +++ bridgedb/metrics.py | 199 ++++++++++++++++++++++--- bridgedb/test/test_distributors_moat_server.py | 8 +- bridgedb/test/test_metrics.py | 132 ++++++++++++++++ doc/bridgedb-metrics-spec.txt | 92 ++++++++---- requirements.txt | 1 + 10 files changed, 455 insertions(+), 79 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 10012ce..8939ac4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ + * FIXES https://bugs.torproject.org/31422 + Make BridgeDB report internal metrics, like the median number of users that + bridges were handed out to. + * FIXES https://bugs.torproject.org/34260 Parse bridge blocking information from SQL database. diff --git a/bridgedb/distributors/email/autoresponder.py b/bridgedb/distributors/email/autoresponder.py index 9b0ac53..94a8f75 100644 --- a/bridgedb/distributors/email/autoresponder.py +++ b/bridgedb/distributors/email/autoresponder.py @@ -62,9 +62,10 @@ from bridgedb.parse.addr import canonicalizeEmailDomain from bridgedb.util import levenshteinDistance from bridgedb import translations -# We use our metrics singleton to keep track of BridgeDB metrics such as +# We use our metrics singletons to keep track of BridgeDB metrics such as # "number of failed HTTPS bridge requests." -metrix = metrics.EmailMetrics() +emailMetrix = metrics.EmailMetrics() +internalMetrix = metrics.InternalMetrics() def createResponseBody(lines, context, client, lang='en'): @@ -113,6 +114,9 @@ def createResponseBody(lines, context, client, lang='en'): transport = bridgeRequest.justOnePTType() answer = "".join(" %s\r\n" % b.getBridgeLine( bridgeRequest, context.includeFingerprints) for b in bridges) + internalMetrix.recordHandoutsPerBridge(bridgeRequest, bridges) + else: + internalMetrix.recordEmptyEmailResponse() return templates.buildAnswerMessage(translator, client, answer) def generateResponse(fromAddress, client, body, subject=None, @@ -396,9 +400,9 @@ class SMTPAutoresponder(smtp.SMTPClient): # request. translator = translations.installTranslations(lang) if body is not None and translator.gettext(strings.EMAIL_MISC_TEXT[1]) in body: - metrix.recordValidEmailRequest(self) + emailMetrix.recordValidEmailRequest(self) else: - metrix.recordInvalidEmailRequest(self) + emailMetrix.recordInvalidEmailRequest(self) if not body: return # The client was already warned. diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index 91757e7..fd3fc74 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -90,9 +90,10 @@ logging.debug("Set template root to %s" % TEMPLATE_DIR) #: A list of supported language tuples. Use getSortedLangList() to read this variable. supported_langs = [] -# We use our metrics singleton to keep track of BridgeDB metrics such as +# We use our metrics singletons to keep track of BridgeDB metrics such as # "number of failed HTTPS bridge requests." -metrix = metrics.HTTPSMetrics() +httpsMetrix = metrics.HTTPSMetrics() +internalMetrix = metrics.InternalMetrics() def stringifyRequestArgs(args): @@ -574,7 +575,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): try: if self.checkSolution(request) is True: - metrix.recordValidHTTPSRequest(request) + httpsMetrix.recordValidHTTPSRequest(request) return self.resource.render(request) except ValueError as err: logging.debug(str(err)) @@ -584,14 +585,14 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): # work of art" as pennance for their sins. d = task.deferLater(reactor, 1, lambda: request) d.addCallback(redirectMaliciousRequest) - metrix.recordInvalidHTTPSRequest(request) + httpsMetrix.recordInvalidHTTPSRequest(request) return NOT_DONE_YET except Exception as err: logging.debug(str(err)) - metrix.recordInvalidHTTPSRequest(request) + httpsMetrix.recordInvalidHTTPSRequest(request) return replaceErrorPage(request, err) - metrix.recordInvalidHTTPSRequest(request) + httpsMetrix.recordInvalidHTTPSRequest(request) logging.debug("Client failed a CAPTCHA; returning redirect to %s" % request.uri) return redirectTo(request.uri, request) @@ -847,12 +848,12 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): # breaking). Hence, the 'no cover' pragma. if solution.is_valid: # pragma: no cover logging.info("Valid CAPTCHA solution from %r." % clientIP) - metrix.recordValidHTTPSRequest(request) + httpsMetrix.recordValidHTTPSRequest(request) return (True, request) else: logging.info("Invalid CAPTCHA solution from %r: %r" % (clientIP, solution.error_code)) - metrix.recordInvalidHTTPSRequest(request) + httpsMetrix.recordInvalidHTTPSRequest(request) return (False, request) d = txrecaptcha.submit(challenge, response, self.secretKey, @@ -1000,6 +1001,8 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): bridgeLines = [replaceControlChars(bridge.getBridgeLine( bridgeRequest, self.includeFingerprints)) for bridge in bridges] + internalMetrix.recordHandoutsPerBridge(bridgeRequest, bridges) + if antibot.isRequestFromBot(request): transports = bridgeRequest.transports # Return either a decoy bridge or no bridge. @@ -1059,6 +1062,9 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): rtl = False format = self.getResponseFormat(request) + if not bridgeLines: + internalMetrix.recordEmptyHTTPSResponse() + if format == 'plain': request.setHeader("Content-Type", "text/plain") try: diff --git a/bridgedb/distributors/moat/server.py b/bridgedb/distributors/moat/server.py index cdf2022..7e83d94 100644 --- a/bridgedb/distributors/moat/server.py +++ b/bridgedb/distributors/moat/server.py @@ -51,9 +51,10 @@ from bridgedb.schedule import Unscheduled from bridgedb.schedule import ScheduledInterval from bridgedb.util import replaceControlChars -# We use our metrics singleton to keep track of BridgeDB metrics such as +# We use our metrics singletons to keep track of BridgeDB metrics such as # "number of failed HTTPS bridge requests." -metrix = metrics.MoatMetrics() +moatMetrix = metrics.MoatMetrics() +internalMetrix = metrics.InternalMetrics() #: The current version of the moat JSON API that we speak @@ -538,26 +539,37 @@ class CaptchaCheckResource(CaptchaResource): return bridgeRequest - def getBridgeLines(self, bridgeRequest): - """Get bridge lines for a client's HTTP request. + def getBridges(self, bridgeRequest): + """Get bridges for a client's HTTP request. :type bridgeRequest: :class:`MoatBridgeRequest` :param bridgeRequest: A valid bridge request object with pre-generated filters (as returned by :meth:`createBridgeRequest`). :rtype: list - :returns: A list of bridge lines. + :return: A list of :class:`~bridgedb.bridges.Bridge`s. """ - bridgeLines = list() + bridges = list() interval = self.schedule.intervalStart(time.time()) logging.debug("Replying to JSON API request from %s." % bridgeRequest.client) if bridgeRequest.isValid(): bridges = self.distributor.getBridges(bridgeRequest, interval) - bridgeLines = [replaceControlChars(bridge.getBridgeLine(bridgeRequest)) - for bridge in bridges] - return bridgeLines + return bridges + + def getBridgeLines(self, bridgeRequest, bridges): + """ + :type bridgeRequest: :class:`MoatBridgeRequest` + :param bridgeRequest: A valid bridge request object with pre-generated + filters (as returned by :meth:`createBridgeRequest`). + :param list bridges: A list of :class:`~bridgedb.bridges.Bridge` + objects. + :rtype: list + :return: A list of bridge lines. + """ + return [replaceControlChars(bridge.getBridgeLine(bridgeRequest)) + for bridge in bridges] def extractClientSolution(self, data): """Extract the client's CAPTCHA solution from a POST request. @@ -686,7 +698,7 @@ class CaptchaCheckResource(CaptchaResource): if error: # pragma: no cover logging.debug("Error while checking moat request headers.") - metrix.recordInvalidMoatRequest(request) + moatMetrix.recordInvalidMoatRequest(request) return error.render(request) data = { @@ -714,25 +726,30 @@ class CaptchaCheckResource(CaptchaResource): valid = self.checkSolution(challenge, solution, clientIP) except captcha.CaptchaExpired: logging.debug("The challenge had timed out") - metrix.recordInvalidMoatRequest(request) + moatMetrix.recordInvalidMoatRequest(request) return self.failureResponse(5, request) except Exception as impossible: logging.warn("Unhandled exception while processing a POST /fetch request!") logging.error(impossible) - metrix.recordInvalidMoatRequest(request) + moatMetrix.recordInvalidMoatRequest(request) return self.failureResponse(4, request) if valid: qrcode = None bridgeRequest = self.createBridgeRequest(clientIP, client_data) - bridgeLines = self.getBridgeLines(bridgeRequest) - metrix.recordValidMoatRequest(request) + bridges = self.getBridges(bridgeRequest) + bridgeLines = self.getBridgeLines(bridgeRequest, bridges) + moatMetrix.recordValidMoatRequest(request) # If we can only return less than the configured # MOAT_BRIDGES_PER_ANSWER then log a warning. if len(bridgeLines) < self.nBridgesToGive: logging.warn(("Not enough bridges of the type specified to " "fulfill the following request: %s") % bridgeRequest) + if not bridgeLines: + internalMetrix.recordEmptyMoatResponse() + else: + internalMetrix.recordHandoutsPerBridge(bridgeRequest, bridges) if antibot.isRequestFromBot(request): ttype = transport or "vanilla" @@ -754,7 +771,7 @@ class CaptchaCheckResource(CaptchaResource): return self.formatDataForResponse(data, request) else: - metrix.recordInvalidMoatRequest(request) + moatMetrix.recordInvalidMoatRequest(request) return self.failureResponse(4, request) diff --git a/bridgedb/main.py b/bridgedb/main.py index 8b851a3..8fdec23 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -483,6 +483,23 @@ def run(options, reactor=reactor): else: logging.warn("No Moat distributor created!") + metrix = metrics.InternalMetrics() + logging.info("Logging bridge ring metrics for %d rings." % + len(hashring.ringsByName)) + for ringName, ring in hashring.ringsByName.items(): + # Ring is of type FilteredBridgeSplitter or UnallocatedHolder. + # FilteredBridgeSplitter splits bridges into subhashrings based on + # filters. + if hasattr(ring, "filterRings"): + for (ringname, (filterFn, subring)) in ring.filterRings.items(): + subRingName = "-".join(ring.extractFilterNames(ringname)) + metrix.recordBridgesInHashring(ringName, + subRingName, + len(subring)) + elif hasattr(ring, "fingerprints"): + metrix.recordBridgesInHashring(ringName, "unallocated", + len(ring.fingerprints)) + # Dump bridge pool assignments to disk. writeAssignments(hashring, state.ASSIGNMENTS_FILE) state.save() diff --git a/bridgedb/metrics.py b/bridgedb/metrics.py index 48713f0..14073ee 100644 --- a/bridgedb/metrics.py +++ b/bridgedb/metrics.py @@ -18,6 +18,8 @@ import ipaddr import operator import json import datetime +import statistics +import numpy from bridgedb import geo from bridgedb.distributors.common.http import getClientIP @@ -54,7 +56,7 @@ SUPPORTED_TRANSPORTS = None # Version number for our metrics format. We increment the version if our # format changes. -METRICS_VERSION = 1 +METRICS_VERSION = 2 def setProxies(proxies): @@ -106,14 +108,14 @@ def export(fh, measurementInterval): and dump our metrics. """ - httpsMetrix = HTTPSMetrics() - emailMetrix = EmailMetrics() - moatMetrix = MoatMetrics() + metrics = [HTTPSMetrics(), + EmailMetrics(), + MoatMetrics(), + InternalMetrics()] # Rotate our metrics. - httpsMetrix.rotate() - emailMetrix.rotate() - moatMetrix.rotate() + for m in metrics: + m.rotate() numProxies = len(PROXIES) if PROXIES is not None else 0 if numProxies == 0: @@ -127,17 +129,14 @@ def export(fh, measurementInterval): measurementInterval)) fh.write("bridgedb-metrics-version %d\n" % METRICS_VERSION) - httpsLines = httpsMetrix.getMetrics() - for line in httpsLines: - fh.write("bridgedb-metric-count %s\n" % line) + for m in metrics: + distLines = m.getMetrics() + for line in distLines: + fh.write("bridgedb-metric-count %s\n" % line) + logging.debug("Writing metrics line to file: %s" % line) - moatLines = moatMetrix.getMetrics() - for line in moatLines: - fh.write("bridgedb-metric-count %s\n" % line) - - emailLines = emailMetrix.getMetrics() - for line in emailLines: - fh.write("bridgedb-metric-count %s\n" % line) + for m in metrics: + m.reset() def resolveCountryCode(ipAddr): @@ -200,6 +199,7 @@ class Metrics(metaclass=Singleton): # that, our hot metrics turn into cold metrics, and we start over. self.hotMetrics = dict() self.coldMetrics = dict() + self.unsanitisedSet = set() def rotate(self): """Rotate our metrics.""" @@ -216,8 +216,15 @@ class Metrics(metaclass=Singleton): return anomaly - def getMetrics(self): - """Get our sanitized current metrics, one per line. + def doNotSanitise(self, key): + """ + :param str key: A key that will not be sanitised when exporting our + metrics. + """ + self.unsanitisedSet.add(key) + + def getMetrics(self, sanitized=True): + """Get our (sanitized) current metrics, one per line. Metrics are of the form: @@ -227,15 +234,19 @@ class Metrics(metaclass=Singleton): ... ] + :param bool sanitized: ``True`` if the metrics must be sanitized. :rtype: list :returns: A list of metric lines. """ lines = [] for key, value in self.coldMetrics.items(): - # Round up our value to the nearest multiple of self.binSize to - # reduce the accuracy of our real values. - if (value % self.binSize) > 0: - value += self.binSize - (value % self.binSize) + + # There's no need to sanitize internal metrics. + if sanitized and not key in self.unsanitisedSet: + # Round up our value to the nearest multiple of self.binSize to + # reduce the accuracy of our real values. + if (value % self.binSize) > 0: + value += self.binSize - (value % self.binSize) lines.append("%s %d" % (key, value)) return lines @@ -290,6 +301,148 @@ class Metrics(metaclass=Singleton): return key + def reset(self): + """Reset internal variables after a metrics interval.""" + pass + + +class InternalMetrics(Metrics): + + def __init__(self): + super(InternalMetrics, self).__init__() + self.keyPrefix = "internal" + # Maps bridges to the number of time they have been handed out. + self.bridgeHandouts = {} + + # There's no reason for the following metrics to be sanitised. + handoutsPrefix = "{}.handouts".format(self.keyPrefix) + self.doNotSanitise("{}.unique-bridges".format(handoutsPrefix)) + self.doNotSanitise("{}.median".format(handoutsPrefix)) + self.doNotSanitise("{}.min".format(handoutsPrefix)) + self.doNotSanitise("{}.max".format(handoutsPrefix)) + self.doNotSanitise("{}.quartile1".format(handoutsPrefix)) + self.doNotSanitise("{}.quartile3".format(handoutsPrefix)) + self.doNotSanitise("{}.lower-whisker".format(handoutsPrefix)) + self.doNotSanitise("{}.upper-whisker".format(handoutsPrefix)) + + def reset(self): + """Reset bridge handouts after each interval.""" + + # Log the bridge that has seen the most handouts. This helps us + # understand BridgeDB better. + items = self.bridgeHandouts.items() + if len(items): + bridgeLine, num = sorted(items, key=lambda x: x[1], + reverse=True)[0] + logging.debug("Bridge line with most handouts (%d): %s" % + (num, bridgeLine)) + + self.bridgeHandouts = {} + + def _recordEmptyResponse(self, distributor): + """ + Record an empty bridge request response for the given distributor. + + :param str distributor: A bridge distributor, e.g., "https". + """ + self.inc("{}.{}.empty-response".format(self.keyPrefix, distributor)) + + def recordEmptyEmailResponse(self): + self._recordEmptyResponse("email") + + def recordEmptyHTTPSResponse(self): + self._recordEmptyResponse("https") + + def recordEmptyMoatResponse(self): + self._recordEmptyResponse("moat") + + def recordHandoutsPerBridge(self, bridgeRequest, bridges): + """ + Record how often a given bridge was handed out. + + Note that bridges that were not handed out will not be part of these + metrics. + + :type bridgeRequest: :api:`bridgerequest.BridgeRequestBase` + :param bridgeRequest: A bridge request for either one of our + distributors. + :param list bridges: A list of :class:`~bridgedb.Bridges.Bridge`s. + """ + + handoutsPrefix = "{}.handouts".format(self.keyPrefix) + + if bridgeRequest is None or bridges is None: + logging.warning("Given bridgeRequest and bridges cannot be None.") + return + + # Keep track of how many IPv4 and IPv6 requests we are seeing. + ipVersion = bridgeRequest.ipVersion + if ipVersion not in [4, 6]: + logging.warning("Got bridge request for unsupported IP version " + "{}.".format(ipVersion)) + return + else: + self.inc("{}.ipv{}".format(handoutsPrefix, ipVersion)) + + # Keep track of how many times we're handing out a given bridge. + for bridge in bridges: + # Use bridge lines as dictionary key. We cannot use the bridge + # objects because BridgeDB reloads its descriptors every 30 + # minutes, at which points the bridge objects change. + key = bridge.getBridgeLine(bridgeRequest) + num = self.bridgeHandouts.get(key, None) + if num is None: + self.bridgeHandouts[key] = 1 + else: + self.bridgeHandouts[key] = num + 1 + + # We need more than two handouts to calculate our statistics. + values = self.bridgeHandouts.values() + if len(values) <= 2: + return + + # Update our statistics. + self.set("{}.median".format(handoutsPrefix), + statistics.median(values)) + self.set("{}.min".format(handoutsPrefix), min(values)) + self.set("{}.max".format(handoutsPrefix), max(values)) + self.set("{}.unique-bridges".format(handoutsPrefix), + len(self.bridgeHandouts)) + # Python 3.8 comes with a statistics.quantiles function, which we + # should use instead of numpy once 3.8 is available in Debian stable. + q1, q3 = numpy.quantile(numpy.array(list(values)), [0.25, 0.75]) + self.set("{}.quartile1".format(handoutsPrefix), q1) + self.set("{}.quartile3".format(handoutsPrefix), q3) + # Determine our inter-quartile range (the difference between quartile 3 + # and quartile 1) and use it to calculate the upper and lower whiskers + # as you would see them in a boxplot. + iqr = q3 - q1 + lowerWhisker = min([x for x in values if x >= q1 - (1.5 * iqr)]) + upperWhisker = max([x for x in values if x <= q3 + (1.5 * iqr)]) + self.set("{}.lower-whisker".format(handoutsPrefix), lowerWhisker) + self.set("{}.upper-whisker".format(handoutsPrefix), upperWhisker) + + def recordBridgesInHashring(self, ringName, subRingName, numBridges): + """ + Record the number of bridges per hashring. + + :param str ringName: The name of the ring, e.g., "https". + :param str subRingName: The name of the subring, e.g., + "byIPv6-bySubring1of4". + :param int numBridges: The number of bridges in the given subring. + """ + + if not ringName or not subRingName: + logging.warning("Ring name ({}) and subring name ({}) cannot be " + "empty.".format(ringName, subRingName)) + return + + logging.info("Recording metrics for bridge (sub)rings: %s/%s/%d." % + (ringName, subRingName, numBridges)) + # E.g, concatenate "https" with "byipv6-bysubring1of4". + key = "{}.{}.{}".format(self.keyPrefix, ringName, subRingName.lower()) + self.set(key, numBridges) + class HTTPSMetrics(Metrics): diff --git a/bridgedb/test/test_distributors_moat_server.py b/bridgedb/test/test_distributors_moat_server.py index 695116b..3d33823 100644 --- a/bridgedb/test/test_distributors_moat_server.py +++ b/bridgedb/test/test_distributors_moat_server.py @@ -586,7 +586,7 @@ class MockCaptchaCheckResource(server.CaptchaCheckResource): """A mocked :class:`server.CaptchaCheckResource` whose ``getBridgeLines`` method returns no bridges. """ - def getBridgeLines(self, bridgeRequest): + def getBridgeLines(self, bridgeRequest, bridges): return list() @@ -860,7 +860,8 @@ class CaptchaCheckResourceTests(unittest.TestCase): content = json.loads(encoded_content)['data'][0] bridgeRequest = self.resource.createBridgeRequest('3.3.3.3', content) - bridgelines = self.resource.getBridgeLines(bridgeRequest) + bridges = self.resource.getBridges(bridgeRequest) + bridgelines = self.resource.getBridgeLines(bridgeRequest, bridges) self.assertTrue(bridgelines) @@ -869,7 +870,8 @@ class CaptchaCheckResourceTests(unittest.TestCase): request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) bridgeRequest = self.resource.createBridgeRequest('3.3.3.3', None) - bridgelines = self.resource.getBridgeLines(bridgeRequest) + bridges = self.resource.getBridges(bridgeRequest) + bridgelines = self.resource.getBridgeLines(bridgeRequest, bridges) self.assertFalse(bridgeRequest.isValid()) self.assertEqual(len(bridgelines), 0) diff --git a/bridgedb/test/test_metrics.py b/bridgedb/test/test_metrics.py index fcc9e4a..12b3ee1 100644 --- a/bridgedb/test/test_metrics.py +++ b/bridgedb/test/test_metrics.py @@ -18,13 +18,17 @@ functioning as expected. import io import json import os +import copy from bridgedb import metrics +from bridgedb.test import util from bridgedb.test.https_helpers import DummyRequest from bridgedb.distributors.email.server import SMTPMessage +from bridgedb.distributors.https.server import HTTPSBridgeRequest from bridgedb.test.email_helpers import _createMailServerContext from bridgedb.test.email_helpers import _createConfig from bridgedb.distributors.moat import server +from bridgedb.bridges import Bridge from twisted.trial import unittest from twisted.test import proto_helpers @@ -41,6 +45,7 @@ class StateTest(unittest.TestCase): type(metrics.HTTPSMetrics()).clear() type(metrics.EmailMetrics()).clear() type(metrics.MoatMetrics()).clear() + type(metrics.InternalMetrics()).clear() metrics.setSupportedTransports({ 'obfs2': False, @@ -213,3 +218,130 @@ class StateTest(unittest.TestCase): self.assertTrue(metrics.isBridgeTypeSupported("obfs4")) self.assertTrue(metrics.isBridgeTypeSupported("vanilla")) self.assertFalse(metrics.isBridgeTypeSupported("xxx")) + + def test_bridge_handouts(self): + + metrix = metrics.InternalMetrics() + bridges = copy.deepcopy(util.generateFakeBridges()) + bridge1, bridge2, bridge3 = bridges[0:3] + m = metrix.hotMetrics + + br = HTTPSBridgeRequest() + br.withIPversion({"ipv6": "4"}) + br.isValid(True) + + # Record a number of distribution events for three separate bridges. + for i in range(10): + metrix.recordHandoutsPerBridge(br, [bridge1]) + for i in range(5): + metrix.recordHandoutsPerBridge(br, [bridge2]) + metrix.recordHandoutsPerBridge(br, [bridge3]) + + self.assertEqual(m["internal.handouts.unique-bridges"], 3) + self.assertEqual(m["internal.handouts.min"], 1) + self.assertEqual(m["internal.handouts.max"], 10) + self.assertEqual(m["internal.handouts.median"], 5) + + # Internal metrics must not be sanitized. + metrix.rotate() + lines = metrix.getMetrics() + self.assertIn("internal.handouts.unique-bridges 3", lines) + self.assertIn("internal.handouts.median 5", lines) + self.assertIn("internal.handouts.min 1", lines) + self.assertIn("internal.handouts.max 10", lines) + + def test_empty_responses(self): + + metrix = metrics.InternalMetrics() + + # Unlike all other internal metrics, empty responses are sanitized. + for i in range(10): + metrix.recordEmptyEmailResponse() + for i in range(11): + metrix.recordEmptyMoatResponse() + metrix.recordEmptyHTTPSResponse() + + metrix.rotate() + lines = metrix.getMetrics() + + self.assertEqual(len(lines), 3) + self.assertIn("internal.email.empty-response 10", lines) + self.assertIn("internal.moat.empty-response 20", lines) + self.assertIn("internal.https.empty-response 10", lines) + + def test_rings(self): + + metrix = metrics.InternalMetrics() + + # Empty parameters must not be recorded. + metrix.recordBridgesInHashring("", "", 20) + self.assertEqual(len(metrix.hotMetrics), 0) + + metrix.recordBridgesInHashring("https", "byIPv6-bySubring1of4", 20) + self.assertEqual(len(metrix.hotMetrics), 1) + self.assertEqual(list(metrix.hotMetrics.keys()), + ["internal.https.byipv6-bysubring1of4"]) + + def test_ipv4_ipv6_requests(self): + + metrix = metrics.InternalMetrics() + v6Req = HTTPSBridgeRequest() + v6Req.withIPversion({"ipv6": "4"}) + v4Req = HTTPSBridgeRequest() + v4Req.withIPversion({}) + + bridges = copy.deepcopy(util.generateFakeBridges()) + + for i in range(9): + metrix.recordHandoutsPerBridge(v6Req, [bridges[0]]) + metrix.recordHandoutsPerBridge(v6Req, [bridges[1]]) + + for i in range(11): + metrix.recordHandoutsPerBridge(v4Req, [bridges[0]]) + + metrix.rotate() + lines = metrix.getMetrics() + + self.assertIn("internal.handouts.ipv6 10", lines) + self.assertIn("internal.handouts.ipv4 20", lines) + + def test_handouts(self): + + metrix = metrics.InternalMetrics() + metrix.recordHandoutsPerBridge(None, None) + self.assertEqual(len(metrix.hotMetrics), 0) + + req = HTTPSBridgeRequest() + req.withIPversion({}) + req.isValid(True) + bridges = copy.deepcopy(util.generateFakeBridges()) + + metrix.recordHandoutsPerBridge(req, [bridges[0]]) + self.assertNotIn("internal.handouts.median", metrix.hotMetrics.keys()) + metrix.recordHandoutsPerBridge(req, [bridges[1]]) + self.assertNotIn("internal.handouts.median", metrix.hotMetrics.keys()) + metrix.recordHandoutsPerBridge(req, [bridges[2]]) + self.assertEqual(metrix.hotMetrics["internal.handouts.median"], 1) + + metrix.recordHandoutsPerBridge(req, [bridges[1]]) + metrix.recordHandoutsPerBridge(req, [bridges[2]]) + metrix.recordHandoutsPerBridge(req, [bridges[2]]) + self.assertEqual(metrix.hotMetrics["internal.handouts.min"], 1) + self.assertEqual(metrix.hotMetrics["internal.handouts.median"], 2) + self.assertEqual(metrix.hotMetrics["internal.handouts.max"], 3) + self.assertEqual(metrix.hotMetrics["internal.handouts.unique-bridges"], 3) + self.assertEqual(metrix.hotMetrics["internal.handouts.quartile1"], 1.5) + self.assertEqual(metrix.hotMetrics["internal.handouts.quartile3"], 2.5) + self.assertEqual(metrix.hotMetrics["internal.handouts.lower-whisker"], 1) + self.assertEqual(metrix.hotMetrics["internal.handouts.upper-whisker"], 3) + + def test_metrics_reset(self): + metrix = metrics.InternalMetrics() + req = HTTPSBridgeRequest() + req.withIPversion({}) + bridges = copy.deepcopy(util.generateFakeBridges()) + metrix.recordHandoutsPerBridge(req, [bridges[0]]) + + self.assertTrue(len(metrix.bridgeHandouts) > 0) + metrix.reset() + self.assertTrue(len(metrix.bridgeHandouts) == 0) diff --git a/doc/bridgedb-metrics-spec.txt b/doc/bridgedb-metrics-spec.txt index 14c38f9..aac838a 100644 --- a/doc/bridgedb-metrics-spec.txt +++ b/doc/bridgedb-metrics-spec.txt @@ -1,4 +1,4 @@ - BridgeDB metrics (version 1) + BridgeDB metrics (version 2) BridgeDB exports usage metrics once every 24 hours. These metrics encode how many approximate successful/failed requests BridgeDB has seen @@ -33,42 +33,82 @@ file is formatted as follows: "bridgedb-metric-count" METRIC_KEY COUNT NL [Any number.] - METRIC_KEY determines a metrics key, which consists of several - fields, separated by a period: - - DISTRIBUTION "." TRANSPORT "." CC/EMAIL "." "success" | "fail" "." RESERVED - - DISTRIBUTION is BridgeDB's distribution mechanism, which includes - "https", "email", and "moat". These distribution mechanisms may - change in the future. + METRIC_KEY specifies a metrics key, which consists of several fields, + separated by a period. These fields form a hierarchy. At the root of + the hierarchy are our three distribution mechanisms ("https", "email", + and "moat") and "internal", which represents BridgeDB-internal metrics. + The hierarchy is of the following form: + + * "https" + └─* TRANSPORT + └─* CC + └─* "success" | "fail" + └─* RESERVED + + * "email" + └─* TRANSPORT + └─* EMAIL + └─* "success" | "fail" + └─* RESERVED + + * "moat" + └─* TRANSPORT + └─* CC + └─* "success" | "fail" + └─* RESERVED + + * "internal" + ├─* "handouts" + │ ├─* "min" + │ ├─* "max" + │ ├─* "median" + │ ├─* "quartile1" + │ ├─* "quartile3" + │ ├─* "lower-whisker" + │ ├─* "upper-whisker" + │ ├─* "ipv4" + │ └─* "ipv6" + │ + └─* TRANSPORT + ├─* "empty-response" + └─* SUBRING + + Strings in between quotes (e.g., "handouts") are literals and show up in + the hierarchy as is. Upper-case strings (e.g., TRANSPORT) are + placeholders, which are explained below. TRANSPORT refers to a pluggable transport protocol. This includes "obfs2", "obfs3", "obfs4", "scramblesuit", and "fte". These pluggable transports will change in the future. - CC/EMAIL refers to a two-letter country code of the user's IP - address iff DISTRIBUTION is "moat" or "https"; or to an email - provider iff DISTRIBUTION is "email". We use two reserved country - codes, "??" and "zz". "??" denotes that we couldn't map an IP - address to its country, e.g., because our geolocation API was - unable to. "zz" denotes a proxy IP address, e.g., Tor exit - relays. The two allowed email providers are "gmail" and "riseup". + CC refers to a two-letter country code of the user's IP address. We use + two reserved country codes, "??" and "zz". "??" denotes that we couldn't + map an IP address to its country, e.g., because our geolocation API was + unable to. "zz" denotes a proxy IP address, e.g., Tor exit relays. - The next field is either "success" or "fail", depending on if the - BridgeDB request was successful or not. A request is successful - if BridgeDB attempts to provide the user with bridges, even if - BridgeDB currently has no bridges available. A request has failed - if BridgeDB won't provide the user with bridges, for example, if - the user could not solve the CAPTCHA. + EMAIL refers to an email provider. The two currently-supported email + providers are "gmail" and "riseup". - The field RESERVED is reserved for an anomaly score. It is - currently set to "none" and should be ignored by implementations. + The field RESERVED is reserved for an anomaly score. It is currently set + to "none" and should be ignored by implementations. COUNT is the approximate number of user requests for the given METRIC_KEY. We round up the number of requests to the next - multiple of 10 to preserve some user privacy. + multiple of 10 to preserve some user privacy. Some metrics key are not + rounded up to the next multiple of ten because they are not sensitive. + + One label either takes on the value "success" or "fail", depending on if + the BridgeDB request was successful or not. A request is successful if + BridgeDB attempts to provide the user with bridges, even if BridgeDB + currently has no bridges available. A request has failed if BridgeDB + won't provide the user with bridges, for example, if the user could not + solve the CAPTCHA. - Examples: + Here are several examples: bridgedb-metric-count https.scramblesuit.zz.fail.none 100 bridgedb-metric-count moat.obfs4.??.success.none 3550 bridgedb-metric-count email.fte.gmail.fail.none 10 + bridgedb-metric-count internal.handouts.ipv4 20 + bridgedb-metric-count internal.moat.empty-response 10 + bridgedb-metric-count internal.handouts.min 23 + bridgedb-metric-count internal.https.byipv6-bysubring1of4 40 diff --git a/requirements.txt b/requirements.txt index fdf1043..08edf9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ qrcode==6.1 service_identity==18.1.0 stem==1.8.0 zope.interface==5.1.0 +numpy==1.18.5

1 0

[bridgedb/master] Merge branch 'bug/40001' into 'develop'
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit 387f2b403abf5ad1a4bec8f8132de5589ecdde23 Merge: 28c5479 2351f86 Author: Philipp Winter <phw(a)torproject.org> Date: Tue Jul 7 16:02:06 2020 +0000 Merge branch 'bug/40001' into 'develop' Remove the --reload command line switch. See merge request tpo/anti-censorship/bridgedb!4 CHANGELOG | 3 +++ README.rst | 4 ---- bridgedb/parse/options.py | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-)

1 0

[bridgedb/master] Remove coverage and pypi link; update Travis CI.
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit c2f4661aa40399efa2d6a596a219cabbd73acae8 Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Jul 7 09:19:01 2020 -0700 Remove coverage and pypi link; update Travis CI. Our Travis CI link still pointed to sysrqb's profile. This commit updates the link to use phw's profile. I'm also removing the (outdated) pypi and coverage links. This fixes tpo/anti-censorship/bridgedb#40002. --- README.rst | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 51eff40..a06c2cd 100644 --- a/README.rst +++ b/README.rst @@ -1,20 +1,14 @@ -********************************************************** -BridgeDB |Latest Version| |Build Status| |Coverage Status| -********************************************************** +*********************** +BridgeDB |Build Status| +*********************** BridgeDB is a collection of backend servers used to distribute `Tor Bridges <https://www.torproject.org/docs/bridges>`__. Currently, it mainly consists of a webserver with `an HTTPS interface <https://bridges.torproject.org>`__, `an email responder <mailto:bridges@torproject.org>`__, and an SQLite database. -.. |Latest Version| image:: https://pypip.in/version/bridgedb/badge.svg?style=flat - :target: https://pypi.python.org/pypi/bridgedb/ -.. |Build Status| image:: https://travis-ci.org/sysrqbci/bridgedb.svg - :target: https://travis-ci.org/sysrqbci/bridgedb -.. |Coverage Status| image:: https://coveralls.io/repos/github/sysrqbci/bridgedb/badge.svg?branch=develop - :target: https://coveralls.io/github/sysrqbci/bridgedb?branch=develop - - +.. |Build Status| image:: https://travis-ci.org/NullHypothesis/bridgedb.svg?branch=master + :target: https://travis-ci.org/github/NullHypothesis/bridgedb .. image:: doc/sphinx/source/_static/bay-bridge.jpg :scale: 80%

1 0

[bridgedb/master] Parse bridge blocking info from SQL database.
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit 5e0ed0af43e876c3305ab75c782ac67b51ea1eb9 Author: Philipp Winter <phw(a)torproject.org> Date: Wed Jul 8 20:23:53 2020 +0000 Parse bridge blocking info from SQL database. The plan currently is that wolpertinger will populate our SQL database with bridge blocking info provided by OONI. This patch adds code that parses this data and adds it to our existing bridge objects. This fixes tpo/anti-censorship/bridgedb#34260. --- CHANGELOG | 3 + bridgedb/Storage.py | 138 ++++++++++++++++++++++++++++++++++++- bridgedb/main.py | 26 +++++++ bridgedb/test/test_Storage.py | 157 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 321 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0351de7..10012ce 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ + * FIXES https://bugs.torproject.org/34260 + Parse bridge blocking information from SQL database. + * FIXES https://gitlab.torproject.org/tpo/anti-censorship/bridgedb/-/issues/40001 Remove the --reload command line switch. It doesn't actually do anything. diff --git a/bridgedb/Storage.py b/bridgedb/Storage.py index 2859cf1..4ae3c9e 100644 --- a/bridgedb/Storage.py +++ b/bridgedb/Storage.py @@ -12,6 +12,7 @@ from functools import wraps from ipaddr import IPAddress from contextlib import contextmanager import sys +import datetime from bridgedb.Stability import BridgeHistory import threading @@ -19,6 +20,7 @@ import threading toHex = binascii.b2a_hex fromHex = binascii.a2b_hex HEX_ID_LEN = 40 +BRIDGE_REACHABLE, BRIDGE_BLOCKED = 0, 1 def _escapeValue(v): return "'%s'" % v.replace("'", "''") @@ -68,7 +70,7 @@ SCHEMA2_SCRIPT = """ CREATE INDEX EmailedBridgesWhenMailed on EmailedBridges ( email ); - CREATE TABLE BlockedBridges ( + CREATE TABLE BridgeMeasurements ( id INTEGER PRIMARY KEY NOT NULL, hex_key, bridge_type, @@ -77,10 +79,11 @@ SCHEMA2_SCRIPT = """ blocking_country, blocking_asn, measured_by, - last_measured + last_measured, + verdict INTEGER ); - CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key); + CREATE INDEX BlockedBridgesBlockingCountry on BridgeMeasurements(hex_key); CREATE TABLE WarnedEmails ( email PRIMARY KEY NOT NULL, @@ -242,6 +245,34 @@ class Database(object): return retBridges + def getBlockedBridges(self): + """Return a dictionary of bridges that are blocked. + + :rtype: dict + :returns: A dictionary that maps bridge fingerprints (as strings) to a + three-tuple that captures its blocking state: (country, address, + port). + """ + ms = self.__fetchBridgeMeasurements() + return getBlockedBridgesFromSql(ms) + + def __fetchBridgeMeasurements(self): + """Return all bridge measurement rows from the last three years. + + We limit our search to three years for performance reasons because the + bridge measurement table keeps growing and therefore slowing down + queries. + + :rtype: list + :returns: A list of tuples. + """ + cur = self._cur + old_year = datetime.datetime.utcnow() - datetime.timedelta(days=365*3) + cur.execute("SELECT * FROM BridgeMeasurements WHERE last_measured > " + "'%s' ORDER BY blocking_country DESC" % + old_year.strftime("%Y-%m-%d")) + return cur.fetchall() + def getBridgesForDistributor(self, distributor): """Return a list of BridgeData value classes of all bridges in the database that are allocated to distributor 'distributor' @@ -352,6 +383,107 @@ _LOCKED = 0 _OPENED_DB = None _REFCOUNT = 0 +class BridgeMeasurement(object): + def __init__(self, id, fingerprint, bridge_type, address, port, + country, asn, measured_by, last_measured, verdict): + self.fingerprint = fingerprint + self.country = country + self.address = address + self.port = port + try: + self.date = datetime.datetime.strptime(last_measured, "%Y-%m-%d") + except ValueError: + logging.error("Could not convert SQL date string '%s' to " + "datetime object." % last_measured) + self.date = datetime.datetime(1970, 1, 1, 0, 0) + self.verdict = verdict + + def compact(self): + return (self.country, self.address, self.port) + + def __contains__(self, item): + return (self.country == item.country and + self.address == item.address and + self.port == item.port) + + def newerThan(self, other): + return self.date > other.date + + def conflicts(self, other): + return (self.verdict != other.verdict and + self.country == other.country and + self.address == other.address and + self.port == other.port) + +def getBlockedBridgesFromSql(sql_rows): + """Return a dictionary that maps bridge fingerprints to a list of + bridges that are known to be blocked somewhere. + + :param list sql_rows: A list of tuples. Each tuple represents an SQL row. + :rtype: dict + :returns: A dictionary that maps bridge fingerprints (as strings) to a + three-tuple that captures its blocking state: (country, address, + port). + """ + # Separately keep track of measurements that conclude that a bridge is + # blocked or reachable. + blocked = {} + reachable = {} + + def _shouldSkip(m1): + """Return `True` if we can skip this measurement.""" + # Use our 'reachable' dictionary if our original measurement says that + # a bridge is blocked, and vice versa. The purpose is to process + # measurements that are possibly conflicting with the one at hand. + d = reachable if m1.verdict == BRIDGE_BLOCKED else blocked + maybe_conflicting = d.get(m1.fingerprint, None) + if maybe_conflicting is None: + # There is no potentially conflicting measurement. + return False + + for m2 in maybe_conflicting: + if m1.compact() != m2.compact(): + continue + # Conflicting measurement. If m2 is newer than m1, we believe m2. + if m2.newerThan(m1): + return True + # Conflicting measurement. If m1 is newer than m2, we believe m1, + # and remove m1. + if m1.newerThan(m2): + d[m1.fingerprint].remove(m2) + # If we're left with an empty list, get rid of the dictionary + # key altogether. + if len(d[m1.fingerprint]) == 0: + del d[m1.fingerprint] + return False + return False + + for fields in sql_rows: + m = BridgeMeasurement(*fields) + if _shouldSkip(m): + continue + + d = blocked if m.verdict == BRIDGE_BLOCKED else reachable + other_measurements = d.get(m.fingerprint, None) + if other_measurements is None: + # We're dealing with the first "blocked" or "reachable" measurement + # for the given bridge fingerprint. + d[m.fingerprint] = [m] + else: + # Do we have an existing measurement that agrees with the given + # measurement? + if m in other_measurements: + d[m.fingerprint] = [m if m.compact() == x.compact() and + m.newerThan(x) else x for x in other_measurements] + # We're dealing with a new measurement. Add it to the list. + else: + d[m.fingerprint] = other_measurements + [m] + + # Compact-ify the measurements in our dictionary. + for k, v in blocked.items(): + blocked[k] = [i.compact() for i in v] + return blocked + def clearGlobalDB(): """Start from scratch. diff --git a/bridgedb/main.py b/bridgedb/main.py index 44d0668..8b851a3 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -308,6 +308,31 @@ def createBridgeRings(cfg, proxyList, key): return hashring, emailDistributor, ipDistributor, moatDistributor +def loadBlockedBridges(hashring): + """Load bridge blocking info from our SQL database and add it to bridge + objects.""" + + blockedBridges = {} + with bridgedb.Storage.getDB() as db: + blockedBridges = db.getBlockedBridges() + + num_blocked = 0 + for name, ring in hashring.ringsByName.items(): + if name == "unallocated": + continue + for _, bridge in ring.bridges.items(): + l = [] + try: + l = blockedBridges[bridge.fingerprint] + except KeyError: + continue + for blocking_country, address, port in l: + bridge.setBlockedIn(blocking_country, address, port) + num_blocked += 1 + + logging.info("Loaded blocking info for %d bridges.".format(num_blocked)) + + def run(options, reactor=reactor): """This is BridgeDB's main entry point and main runtime loop. @@ -441,6 +466,7 @@ def run(options, reactor=reactor): logging.info("Reparsing bridge descriptors...") load(state, hashring, clear=False) logging.info("Bridges loaded: %d" % len(hashring)) + loadBlockedBridges(hashring) if emailDistributorTmp is not None: emailDistributorTmp.prepopulateRings() # create default rings diff --git a/bridgedb/test/test_Storage.py b/bridgedb/test/test_Storage.py index 720afdd..a1eb7bd 100644 --- a/bridgedb/test/test_Storage.py +++ b/bridgedb/test/test_Storage.py @@ -4,6 +4,7 @@ import os import threading import time +import datetime from twisted.python import log from twisted.trial import unittest @@ -11,6 +12,8 @@ from twisted.internet import reactor from twisted.internet.threads import deferToThread import bridgedb.Storage as Storage +import bridgedb.main as main +from bridgedb.bridges import Bridge from bridgedb.test.util import generateFakeBridges @@ -113,3 +116,157 @@ class DatabaseTest(unittest.TestCase): with Storage.getDB() as db: ringname = db.getBridgeDistributor(bridge, self.validRings) self.assertEqual(ringname, "unallocated") + + def test_BridgeMeasurementComparison(self): + m1 = Storage.BridgeMeasurement(0, "", "", "", "", "", "", "", + "2020-06-17", 0) + m2 = Storage.BridgeMeasurement(0, "", "", "", "", "", "", "", + "2020-06-18", 0) + self.assertTrue(m2.newerThan(m1)) + self.assertFalse(m1.newerThan(m2)) + self.assertFalse(m1.newerThan(m1)) + + def test_BridgeMeasurementCompact(self): + m = Storage.BridgeMeasurement(0, "FINGERPRINT", "obfs4", "1.2.3.4", + "1234", "ru", "1234", "ooni", + "2020-06-17", 0) + self.assertEquals(m.compact(), ("ru", "1.2.3.4", "1234")) + + def test_fetchBridgeMeasurements(self): + + query = "INSERT INTO BridgeMeasurements (hex_key, bridge_type, " \ + "address, port, blocking_country, blocking_asn, " \ + "measured_by, last_measured, verdict) VALUES ('key', " \ + "'obfs4', '1.2.3.4', '1234', 'RU', '1234', 'OONI', '%s', 1)" + oldMsmt = query % "2017-01-01" + newMsmt = query % datetime.datetime.utcnow().strftime("%Y-%m-%d") + + Storage.initializeDBLock() + with Storage.getDB() as db: + db._cur.execute(oldMsmt) + # We're calling _Database__fetchBridgeMeasurements instead of + # __fetchBridgeMeasurements to account for Python's name meddling. + rows = db._Database__fetchBridgeMeasurements() + # Outdated measurements should not be returned. + self.assertEquals(len(rows), 0) + + db._cur.execute(newMsmt) + rows = db._Database__fetchBridgeMeasurements() + # Measurements that are "young enough" should be returned. + self.assertEquals(len(rows), 1) + + def test_main_loadBlockedBridges(self): + Storage.initializeDBLock() + + # Mock configuration object that we use to initialize our bridge rings. + class Cfg(object): + def __init__(self): + self.FORCE_PORTS = [(443, 1)] + self.FORCE_FLAGS = [("Stable", 1)] + self.MOAT_DIST = False + self.HTTPS_DIST = True + self.HTTPS_SHARE = 10 + self.N_IP_CLUSTERS = 1 + self.EMAIL_DIST = False + self.RESERVED_SHARE = 0 + + bridge = self.fakeBridges[0] + addr, port, _ = bridge.orAddresses[0] + cc= "de" + + # Mock object that we use to simulate a database connection. + class DummyDB(object): + def __init__(self): + pass + def __enter__(self): + return self + def __exit__(self, type, value, traceback): + pass + def getBlockedBridges(self): + return {bridge.fingerprint: [(cc, addr, port)]} + def getBridgeDistributor(self, bridge, validRings): + return "https" + def insertBridgeAndGetRing(self, bridge, setRing, seenAt, validRings, defaultPool="unallocated"): + return "https" + def commit(self): + pass + + oldObj = Storage.getDB + Storage.getDB = DummyDB + + hashring, _, _, _ = main.createBridgeRings(Cfg(), None, b'key') + hashring.insert(bridge) + + self.assertEqual(len(hashring), 1) + self.assertFalse(bridge.isBlockedIn(cc)) + self.assertFalse(bridge.isBlockedIn("ab")) + self.assertFalse(bridge.addressIsBlockedIn(cc, addr, port)) + + main.loadBlockedBridges(hashring) + + self.assertTrue(bridge.isBlockedIn(cc)) + self.assertFalse(bridge.isBlockedIn("ab")) + self.assertTrue(bridge.addressIsBlockedIn(cc, addr, port)) + + Storage.getDB = oldObj + + def test_getBlockedBridgesFromSql(self): + + elems = [(0, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17", + Storage.BRIDGE_BLOCKED), + (1, "1111111111111111111111111111111111111111", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01", + Storage.BRIDGE_BLOCKED), + (2, "1111111111111111111111111111111111111111", "obfs4", + "1.2.3.4", "4321", "ru", "4321", "ooni", "2020-06-01", + Storage.BRIDGE_BLOCKED), + (3, "1111111111111111111111111111111111111111", "obfs4", + "1.2.3.4", "4321", "ru", "4321", "ooni", "2020-05-01", + Storage.BRIDGE_REACHABLE)] + b = Storage.getBlockedBridgesFromSql(elems) + self.assertEqual(b, {"0000000000000000000000000000000000000000": + [("ru", "1.2.3.4", "1234")], + "1111111111111111111111111111111111111111": + [("ru", "1.2.3.4", "1234"), + ("ru", "1.2.3.4", "4321")]}) + + # If multiple measurements disagree, we believe the newest one. + elems = [(0, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17", + Storage.BRIDGE_BLOCKED), + (1, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01", + Storage.BRIDGE_REACHABLE)] + b = Storage.getBlockedBridgesFromSql(elems) + self.assertEqual(b, {"0000000000000000000000000000000000000000": + [("ru", "1.2.3.4", "1234")]}) + + elems = [(0, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01", + Storage.BRIDGE_BLOCKED), + (1, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17", + Storage.BRIDGE_REACHABLE)] + b = Storage.getBlockedBridgesFromSql(elems) + self.assertTrue(len(b) == 0) + + # Element ordering must not affect the outcome. + elems = [(1, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17", + Storage.BRIDGE_REACHABLE), + (0, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01", + Storage.BRIDGE_BLOCKED)] + b = Storage.getBlockedBridgesFromSql(elems) + self.assertTrue(len(b) == 0) + + # Redundant measurements should be discarded. + elems = [(1, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17", + Storage.BRIDGE_BLOCKED), + (0, "0000000000000000000000000000000000000000", "obfs4", + "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01", + Storage.BRIDGE_BLOCKED)] + b = Storage.getBlockedBridgesFromSql(elems) + self.assertTrue(len(b) == 1)

1 0

[bridgedb/master] Merge branch 'enhancement/34260' into 'develop'
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit ada3bdca87ff853a55e602beeffb4699ba9b2250 Merge: d8b73ec 5e0ed0a Author: Philipp Winter <phw(a)torproject.org> Date: Wed Jul 8 20:23:53 2020 +0000 Merge branch 'enhancement/34260' into 'develop' Parse bridge blocking info from SQL database. See merge request tpo/anti-censorship/bridgedb!7 CHANGELOG | 3 + bridgedb/Storage.py | 138 ++++++++++++++++++++++++++++++++++++- bridgedb/main.py | 26 +++++++ bridgedb/test/test_Storage.py | 157 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 321 insertions(+), 3 deletions(-)

1 0

[bridgedb/master] Merge branch 'release-0.11.0'
by phw＠torproject.org 08 Jul '20

08 Jul '20

commit fb3cfa7edca3db9387bfa999efaa7021d9ce9de2 Merge: c45439c bd32dd7 Author: Philipp Winter <phw(a)nymity.ch> Date: Wed Jul 8 14:03:47 2020 -0700 Merge branch 'release-0.11.0' CHANGELOG | 20 +++ README.rst | 20 +-- bridgedb.conf | 10 ++ bridgedb/Storage.py | 138 +++++++++++++- bridgedb/bridges.py | 15 ++ bridgedb/distributors/email/autoresponder.py | 12 +- bridgedb/distributors/https/server.py | 22 ++- .../https/templates/assets/images/favicon.ico | Bin 0 -> 4286 bytes bridgedb/distributors/https/templates/base.html | 1 + bridgedb/distributors/moat/server.py | 47 +++-- bridgedb/main.py | 50 ++++++ bridgedb/metrics.py | 199 ++++++++++++++++++--- bridgedb/parse/options.py | 2 - bridgedb/parse/versions.py | 29 +++ bridgedb/test/test_Storage.py | 157 ++++++++++++++++ bridgedb/test/test_bridges.py | 27 +++ bridgedb/test/test_distributors_moat_server.py | 8 +- bridgedb/test/test_metrics.py | 132 ++++++++++++++ doc/bridgedb-metrics-spec.txt | 92 +++++++--- requirements.txt | 1 + setup.py | 3 +- 21 files changed, 885 insertions(+), 100 deletions(-)

1 0