May 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Remove bridgedb.Storage.openOrConvertDatabase().
by isis＠torproject.org 01 May '15

01 May '15

commit fbe46fb0ef8103f3cd09842677034899098e7b8a Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 22:52:25 2015 +0000 Remove bridgedb.Storage.openOrConvertDatabase(). The databases were converted *many* years ago. We don't need this cruft anymore. --- lib/bridgedb/Main.py | 2 +- lib/bridgedb/Storage.py | 66 ++--------------------------------------------- 2 files changed, 3 insertions(+), 65 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index d5f4c24..116f827 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -412,7 +412,7 @@ def run(options, reactor=reactor): # Initialize our DB. bridgedb.Storage.initializeDBLock() - db = bridgedb.Storage.openOrConvertDatabase(cfg.DB_FILE + ".sqlite", cfg.DB_FILE) + db = bridgedb.Storage.openDatabase(cfg.DB_FILE + ".sqlite") bridgedb.Storage.setDBFilename(cfg.DB_FILE + ".sqlite") load(state, splitter, clear=False) diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 732d8ab..f9fcf43 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -130,11 +130,8 @@ class BridgeData: self.last_seen = last_seen class Database(object): - def __init__(self, sqlite_fname, db_fname=None): - if db_fname is None: - self._conn = openDatabase(sqlite_fname) - else: - self._conn = openOrConvertDatabase(sqlite_fname, db_fname) + def __init__(self, sqlite_fname): + self._conn = openDatabase(sqlite_fname) self._cur = self._conn.cursor() self.sqlite_fname = sqlite_fname @@ -326,62 +323,6 @@ def openDatabase(sqlite_file): return conn -def openOrConvertDatabase(sqlite_file, db_file): - """Open a sqlite database, converting it from a db file if needed.""" - if os.path.exists(sqlite_file): - return openDatabase(sqlite_file) - - conn = sqlite3.Connection(sqlite_file) - cur = conn.cursor() - cur.executescript(SCHEMA3_SCRIPT) - conn.commit() - - import anydbm - - try: - db = anydbm.open(db_file, 'r') - except anydbm.error: - return conn - - try: - # We handle all the sp| keys first, since other tables have - # dependencies on Bridges. - for k in db.keys(): - v = db[k] - if k.startswith("sp|"): - assert len(k) == 23 - cur.execute("INSERT INTO Bridges ( hex_key, distributor ) " - "VALUES (?, ?)", (toHex(k[3:]),v)) - # Now we handle the other key types. - for k in db.keys(): - v = db[k] - if k.startswith("fs|"): - assert len(k) == 23 - cur.execute("UPDATE Bridges SET first_seen = ? " - "WHERE hex_key = ?", (v, toHex(k[3:]))) - elif k.startswith("ls|"): - assert len(k) == 23 - cur.execute("UPDATE Bridges SET last_seen = ? " - "WHERE hex_key = ?", (v, toHex(k[3:]))) - #elif k.startswith("em|"): - # keys = list(toHex(i) for i in - # bridgedb.Bridges.chopString(v, bridgedb.Bridges.ID_LEN)) - # cur.executemany("INSERT INTO EmailedBridges ( email, id ) " - # "SELECT ?, id FROM Bridges WHERE hex_key = ?", - # [(k[3:],i) for i in keys]) - elif k.startswith("sp|") or k.startswith("em|"): - pass - else: - logging.warn("Unrecognized key %r", k) - except: - conn.rollback() - conn.close() - os.unlink(sqlite_file) - raise - - conn.commit() - return conn - class DBGeneratorContextManager(GeneratorContextManager): """Helper for @contextmanager decorator. @@ -470,9 +411,6 @@ def initializeDBLock(): _LOCK = threading.RLock() assert _LOCK -def checkAndConvertDB(sqlite_file, db_file): - openOrConvertDatabase(sqlite_file, db_file).close() - def setDBFilename(sqlite_fname): global _DB_FNAME _DB_FNAME = sqlite_fname

1 0

[bridgedb/master] Fix whitespace identation for Sphinx docstrings in bridgedb.Bridges.
by isis＠torproject.org 01 May '15

01 May '15

commit 77782beea451270bdf0401c42798be7fbc2002dc Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 19 03:19:33 2015 +0000 Fix whitespace identation for Sphinx docstrings in bridgedb.Bridges. --- lib/bridgedb/Bridges.py | 73 +++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 0239730..191dc08 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -109,54 +109,49 @@ class BridgeRing(BridgeHolder): :type key: bytes :param key: The HMAC key, generated with - :func:`bridgedb.crypto.getKey`. + :func:`~bridgedb.crypto.getKey`. :type answerParameters: :class:`BridgeRingParameters` :param answerParameters: DOCDOC :ivar dict bridges: A dictionary which maps HMAC keys to - :class:`~bridgedb.Bridges.Bridge`s. + :class:`~bridgedb.bridges.Bridge`s. :ivar dict bridgesByID: A dictionary which maps raw hash digests of - bridge ID keys to - :class:`~bridgedb.Bridges.Bridge`s. + bridge ID keys to :class:`~bridgedb.bridges.Bridge`s. :type hmac: callable :ivar hmac: An HMAC function, which uses the **key** parameter to - generate new HMACs for storing, inserting, and retrieving - :class:`~bridgedb.Bridges.Bridge`s within mappings. + generate new HMACs for storing, inserting, and retrieving + :class:`~bridgedb.bridges.Bridge`s within mappings. :ivar bool isSorted: ``True`` if ``sortedKeys`` is currently sorted. :ivar list sortedKeys: A sorted list of all of the HMACs. :ivar str name: A string which identifies this hashring, used mostly - for differentiating this hashring in log messages, but - it is also used for naming subrings. If this hashring - is a subring, the ``name`` will include whatever - distinguishing parameters differentiate that - particular subring (i.e. ``'(port-443 subring)'`` or - ``'(Stable subring)'``) + for differentiating this hashring in log messages, but it is also + used for naming subrings. If this hashring is a subring, the + ``name`` will include whatever distinguishing parameters + differentiate that particular subring (i.e. ``'(port-443 + subring)'`` or ``'(Stable subring)'``) :type subrings: list :ivar subrings: A list of other ``BridgeRing``s, each of which - contains bridges of a particular type. For example, a - subring might contain only ``Bridge``s which have been - given the "Stable" flag, or it might contain only IPv6 - bridges. Each item in this list should be a 4-tuple: + contains bridges of a particular type. For example, a subring + might contain only ``Bridge``s which have been given the "Stable" + flag, or it might contain only IPv6 bridges. Each item in this + list should be a 4-tuple:: - ``(type, value, count, ring)`` + (type, value, count, ring) - where: + where: - * ``type`` is a string which describes what kind of - parameter is used to determine if a ``Bridge`` - belongs in that subring, i.e. ``'port'`` or - ``'flag'``. + - ``type`` is a string which describes what kind of parameter is + used to determine if a ``Bridge`` belongs in that subring, + i.e. ``'port'`` or ``'flag'``. - * ``value`` is a specific value pertaining to the - ``type``, e.g. ``type='port'; value=443``. + - ``value`` is a specific value pertaining to the ``type``, + e.g. ``type='port'; value=443``. - * ``count`` is an integer for the current total - number of bridges in the subring. + - ``count`` is an integer for the current total number of + bridges in the subring. - * ``ring`` is a - :class:`~bridgedb.Bridges.BridgeRing`; it is the - sub hashring which contains ``count`` number of - :class:`~bridgedb.Bridges.Bridge`s of a certain - ``type``. + - ``ring`` is a :class:`BridgeRing`; it is the subhashring which + contains ``count`` number of + :class:`~bridgedb.bridges.Bridge`s of a certain ``type``. """ self.bridges = {} self.bridgesByID = {} @@ -268,10 +263,9 @@ class BridgeRing(BridgeHolder): return them. :param bytes pos: The position to jump to. Any bridges returned will - start at this position in the hashring, if there is - a bridge assigned to that position. Otherwise, - indexing will start at the first position after this - one which has a bridge assigned to it. + start at this position in the hashring, if there is a bridge + assigned to that position. Otherwise, indexing will start at the + first position after this one which has a bridge assigned to it. :param int N: The number of bridges to return. :rtype: list :returns: A list of :class:`~bridgedb.Bridges.Bridge`s. @@ -597,7 +591,7 @@ class FilteredBridgeSplitter(BridgeHolder): :param str ringname: A unique name identifying a sub hashring. :rtype: list :returns: A sorted list of strings, all the function names of the - filters applied to the sub hashring named **ringname**. + filters applied to the sub hashring named **ringname**. """ filterNames = [] @@ -618,14 +612,13 @@ class FilteredBridgeSplitter(BridgeHolder): :param subring: The subring to add. :param str ringname: A unique name for identifying the new subring. :param filterFn: A function whose input is a :class:`Bridge`, and - returns True/False based on some filtration criteria. + returns True/False based on some filtration criteria. :type populate_from: iterable or None :param populate_from: A group of :class:`Bridge`s. If given, the newly - added subring will be populated with these - bridges. + added subring will be populated with these bridges. :rtype: bool :returns: False if there was a problem adding the subring, True - otherwise. + otherwise. """ # XXX I think subring and ringname are switched in this function, or # at least that whatever is passed into this function as as the

1 0

[bridgedb/master] Fix log statement parameter for Stem descriptor class names.
by isis＠torproject.org 01 May '15

01 May '15

commit 7e99a163a15cdc6c66b202aa91cc2fac2c440b52 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 19 03:21:00 2015 +0000 Fix log statement parameter for Stem descriptor class names. --- lib/bridgedb/parse/descriptors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index bec10b3..3f8d48e 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -190,7 +190,7 @@ def deduplicate(descriptors, statistics=False): for descriptor in descriptors: fingerprint = descriptor.fingerprint logging.debug("Deduplicating %s descriptor for router %s" - % (str(descriptor.__class__).rsplit('.', 1)[1], + % (descriptor.__class__.__name__.rsplit('.', 1)[-1], safelog.logSafely(fingerprint))) if fingerprint in duplicates: duplicates[fingerprint].append(descriptor)

1 0

[bridgedb/master] Decrease the time spent in deduplicate() benchmarks.
by isis＠torproject.org 01 May '15

01 May '15

commit a64de660b46c19ddaa17f7cf9688464d09202e1d Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 19 04:51:55 2015 +0000 Decrease the time spent in deduplicate() benchmarks. [noci] It's taking a super long time on some of the CI machines. --- lib/bridgedb/test/test_parse_descriptors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 5ef3f6a..a7b22cf 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -492,7 +492,7 @@ class ParseDescriptorsTests(unittest.TestCase): def test_parse_descriptors_parseExtraInfoFiles_benchmark_100_bridges(self): """Benchmark test for ``b.p.descriptors.parseExtraInfoFiles``.""" print() - for i in range(1, 11): + for i in range(1, 6): descFiles = self.createDuplicatesForBenchmark(b=100, n=i) with Benchmarker(): routers = descriptors.parseExtraInfoFiles(*descFiles) @@ -500,7 +500,7 @@ class ParseDescriptorsTests(unittest.TestCase): def test_parse_descriptors_parseExtraInfoFiles_benchmark_1000_bridges(self): """Benchmark test for ``b.p.descriptors.parseExtraInfoFiles``.""" print() - for i in range(1, 11): + for i in range(1, 6): descFiles = self.createDuplicatesForBenchmark(b=1000, n=i) with Benchmarker(): routers = descriptors.parseExtraInfoFiles(*descFiles) @@ -513,7 +513,7 @@ class ParseDescriptorsTests(unittest.TestCase): "Run it on your own free time.")) print() - for i in range(1, 11): + for i in range(1, 6): descFiles = self.createDuplicatesForBenchmark(b=10000, n=i) with Benchmarker(): routers = descriptors.parseExtraInfoFiles(*descFiles)

1 0

[bridgedb/master] Update CHANGELOG for BridgeDB-0.3.2.
by isis＠torproject.org 01 May '15

01 May '15

commit 251945414b7a7f166b9528b4620d4b51ba7a645a Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 22 01:25:42 2015 +0000 Update CHANGELOG for BridgeDB-0.3.2. --- CHANGELOG | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 8c1bb0e..240794e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,19 @@ -Changes in version 0.3.2 - 2015-03-?? +Changes in version 0.3.2 - 2015-05-01 * FIXES a problem with the calculation of Levenshtein distances between blacklisted email addresses and those on incoming email. This fixes a problem with the fuzzy matching implemented in #9385: https://bugs.torproject.org/9385. + * FIXES #1839 https://bugs.torproject.org/1839 + BridgeDB's distributors now rotate their hashrings at + configurable scheduled intervals. + + * FIXES #4771 https://bugs.torproject.org/4771 + BridgeDB now records which of the HTTPS Distributor's + sub-hashrings are used for clients coming from Tor Exit nodes and + other known proxies. + * FIXES #12504 https://bugs.torproject.org/12504 Which Pluggable Transports BridgeDB distributes is now easily configurable via the bridgedb.conf configuration file. @@ -24,11 +33,31 @@ Changes in version 0.3.2 - 2015-03-?? work. This fixes the issue by excluding broken transports from being distributed to clients. + * FIXES #15522 https://bugs.torproject.org/15522 + For all clients who are coming from IPv6 addresses and are not + using Tor, who go to https://bridges.torproject.org, BridgeDB now + groups these clients together by /32. This "grouping" causes all + IPv6 clients within the same IPv6 /32 to get the same bridges. + Previously, BridgeDB grouped IPv6 clients by /64 (which is + ridiculously small, considering standard IPv6 allocation sizes). + + For all clients who are coming from IPv4 addresses and are not + using Tor, BridgeDB now groups these clients together by /16. + Previously, BridgeDB grouped IPv4 clients by /24. (This latter + change was technically made as part of #4771.) + * FIXES #15464 https://bugs.torproject.org/15464 The setup procedure for creating a BridgeDB Continuous Integration build machine is now simplified and generalised to include build environments like Jenkins, not just TravisCI. + * FIXES #15866 https://bugs.torproject.org/15866 + BridgeDB now ignores nearly all the information in the + networkstatus-bridges file created by the BridgeAuthority. + + * ADDS benchmark tests to BridgeDB's test suite, and some of + BridgeDB's algorithms have been revised to improve their speed. + Changes in version 0.3.1 - 2015-03-24

1 0

[bridgedb/master] Add IGNORE_NETWORKSTATUS config option.
by isis＠torproject.org 01 May '15

01 May '15

commit bb3a3db7d8b2126bc5543286252e5fbafe750c16 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 30 06:02:43 2015 +0000 Add IGNORE_NETWORKSTATUS config option. --- bridgedb.conf | 20 +++++++++++++++++++- lib/bridgedb/configure.py | 4 ++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/bridgedb.conf b/bridgedb.conf index cbdead0..51d9047 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -20,10 +20,14 @@ # # CHANGELOG: # ~~~~~~~~~~ -# Changed in version 0.3.2 - 2015-03-30 +# Changed in version 0.3.2 - 2015-04-30 # * CHANGE to using BridgeDB release versions for bridgedb.conf file versions. # * ADD support for specifying bridge rotation periods via the # EMAIL_ROTATION_PERIOD and HTTPS_ROTATION_PERIOD settings. +# * ADD a new option, IGNORE_NETWORKSTATUS, which controls if BridgeDB cares +# whether or not a bridge was present in the BridgeAuthority's +# networkstatus-bridges file, as well as what information is gleaned from +# the bridge-networkstatus documents within that file. # # Changed in version 0.0.15 - 2015-03-26 # * ADD new SUPPORTED_TRANSPORTS and DEFAULT_TRANSPORT settings. @@ -123,6 +127,20 @@ EXTRA_INFO_FILES = ["cached-extrainfo", "cached-extrainfo.new"] # learning which current bridges are Running, as well as their IPv6 addresses. STATUS_FILE = "networkstatus-bridges" +# If ``True``, then the only information which we obtain from the +# networkstatus file (the ``STATUS_FILE`` option) created by the +# BridgeAuthority are the bridge flags (i.e. "Running", "Stable", etc.), and +# we ignore all errors which have anything to do with networkstatus being +# wrong, out-of-date, or missing. +# +# Otherwise, if ``True``, use the networkstatus to determine which bridges +# should be distributed. +# +# See https://bugs.torproject.org/9380#comment:39, +# https://bugs.torproject.org/11216, and https://bugs.torproject.org/15866. +# +IGNORE_NETWORKSTATUS = True + # Certificate file and private key for the HTTPS Distributor. To create a # self-signed cert, run ``scripts/make-ssl-cert`` it will create these files # in your current directory. diff --git a/lib/bridgedb/configure.py b/lib/bridgedb/configure.py index 2992e1c..0056107 100644 --- a/lib/bridgedb/configure.py +++ b/lib/bridgedb/configure.py @@ -120,6 +120,10 @@ def loadConfig(configFile=None, configCls=None): setting = getattr(config, attr, None) # Default to None setattr(config, attr, setting) + for attr in ["IGNORE_NETWORKSTATUS"]: + setting = getattr(config, attr, True) # Default to True + setattr(config, attr, setting) + for attr in ["FORCE_PORTS", "FORCE_FLAGS", "NO_DISTRIBUTION_COUNTRIES"]: setting = getattr(config, attr, []) # Default to empty lists setattr(config, attr, setting)

1 0

[bridgedb/master] Change default Makefile directive.
by isis＠torproject.org 01 May '15

01 May '15

commit 9b2a72469175d92cca13d6559918ba1b0a8c0749 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 22 01:25:42 2015 +0000 Change default Makefile directive. Conflicts: Makefile --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c6c0442..2c09bbe 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,7 @@ TRIAL:=$(shell which trial) VERSION:=$(shell git describe) -all: - python setup.py build +all: uninstall clean install coverage-test test: python setup.py test

1 0

[bridgedb/master] Add unittests for Bridge.updateFrom*(ignoreNetworkstatus=True).
by isis＠torproject.org 01 May '15

01 May '15

commit f128f46de25361ec51341f850b1fc43de06cbf2b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 22 01:25:42 2015 +0000 Add unittests for Bridge.updateFrom*(ignoreNetworkstatus=True). --- lib/bridgedb/test/test_bridges.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index 5a79d5e..7778b02 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -1029,6 +1029,25 @@ class BridgeTests(unittest.TestCase): self.assertIn((ipaddr.IPAddress('123.34.56.78'), 36488, 4), self.bridge.allVanillaAddresses) + def test_Bridge_updateFromNetworkStatus_ignoreNetworkstatus(self): + """Calling updateFromNetworkStatus([…], ignoreNetworkstatus=True) + should update the Bridge's flags, its fingerprint, and its + descriptorDigest, and nothing else. + """ + self.bridge.updateFromNetworkStatus(self.networkstatus, + ignoreNetworkstatus=True) + + self.assertTrue(self.bridge.flags.running) + self.assertTrue(self.bridge.flags.stable) + self.assertIsNotNone(self.bridge.descriptorDigest) + self.assertEqual(self.bridge.fingerprint, + '2C3225C4805331025E211F4B6E5BF45C333FDD2C') + + self.assertIsNone(self.bridge.nickname) + self.assertIsNone(self.bridge.address) + self.assertIsNone(self.bridge.orPort) + self.assertIsNone(self.bridge.orPort) + def test_Bridge_updateFromServerDescriptor(self): """ """ self.bridge.updateFromNetworkStatus(self.networkstatus) @@ -1046,6 +1065,16 @@ class BridgeTests(unittest.TestCase): self.bridge.updateFromServerDescriptor, self.serverdescriptor) + def test_Bridge_updateFromServerDescriptor_ignoreNetworkstatus_no_networkstatus(self): + """Parsing a server descriptor for a bridge which wasn't included in + the networkstatus document from the BridgeAuthority, when + ignoreNetworkstatus=True, should not raise any warnings. + """ + self.bridge.updateFromServerDescriptor(self.serverdescriptor, + ignoreNetworkstatus=True) + self.assertIsNone(self.bridge.descriptors['networkstatus']) + self.assertIsNotNone(self.bridge.descriptors['server']) + def test_Bridge_verifyExtraInfoSignature_good_signature(self): """Calling _verifyExtraInfoSignature() with a descriptor which has a good signature should return None.

1 0

[bridgedb/master] Merge branch 'release-0.3.2'
by isis＠torproject.org 01 May '15

01 May '15

1 0

[bridgedb/master] Remove duplicate Bridge.Flags.update() during descriptor parsing.
by isis＠torproject.org 01 May '15

01 May '15

commit 69a44c5494d76a330df96dcfd72c8fefe2e76614 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 30 06:11:36 2015 +0000 Remove duplicate Bridge.Flags.update() during descriptor parsing. --- lib/bridgedb/Main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 116f827..44413d2 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -121,7 +121,6 @@ def load(state, splitter, clear=False): for router in networkstatuses: bridge = Bridge() bridge.updateFromNetworkStatus(router) - bridge.flags.update(router.flags) try: bridge.assertOK()

1 0