March 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Refactor Filters.filterBridgesByTransport().
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit adb67e64ab65bee8cca6ff0638ab97772c474bc9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 24 08:59:53 2014 +0000 Refactor Filters.filterBridgesByTransport(). --- lib/bridgedb/Filters.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index 4580c87..b7bac87 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -74,23 +74,34 @@ def filterBridgesByIP6(bridge): setattr(filterBridgesByIP6, "description", "ip=6") def filterBridgesByTransport(methodname, addressClass=None): - if addressClass is None: addressClass = IPv4Address - assert (addressClass) in (IPv4Address, IPv6Address) + if not isinstance(addressClass, (IPv6Address, IPv4Address)): + addressClass = IPv4Address + ruleset = frozenset([methodname, addressClass]) try: return funcs[ruleset] except KeyError: - def f(bridge): + + def _filterByTransport(bridge): for transport in bridge.transports: - # ignore method name case - if isinstance(transport.address, addressClass) and \ - transport.methodname.lower() == methodname.lower(): return True + if isinstance(transport.address, addressClass): + # ignore method name case + if transport.methodname.lower() == methodname.lower(): + return True + else: + logging.debug(("Transport methodname '%s' doesn't match " + "requested methodname: '%s'.") + % (transport.methodname, methodname)) + else: + logging.debug(("Transport %s has incorrect address version " + "(%s).") % (transport, addressClass)) return False - f.__name__ = "filterBridgesByTransport(%s,%s)" % (methodname, - addressClass) - setattr(f, "description", "transport=%s"%methodname) - funcs[ruleset] = f - return f + + _filterByTransport.__name__ = ("filterBridgesByTransport(%s,%s)" + % (methodname, addressClass)) + setattr(_filterByTransport, "description", "transport=%s" % methodname) + funcs[ruleset] = _filterByTransport + return _filterByTransport def filterBridgesByNotBlockedIn(countryCode, addressClass=None, methodname=None): """ if at least one address:port of the selected addressClass and

1 0

[bridgedb/master] Change Bridge.__str__() to identify bridges as the ControlPort does.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 928ca525105146657c0f6addadd56332fefad85d Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 24 08:56:09 2014 +0000 Change Bridge.__str__() to identify bridges as the ControlPort does. --- lib/bridgedb/bridges.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index f612565..71afe3b 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -799,18 +799,19 @@ class Bridge(BridgeBackwardsCompatibility): :data:`nickname```.$``:data:`fingerprint`. """ nickname = self.nickname if self.nickname else 'Unnamed' - separator = '.$' + prefix = '$' + separator = "~" fingerprint = self.fingerprint if safelog.safe_logging: - separator = '.$$' + prefix = '$$' if fingerprint: - fingerprint = hashlib.sha1(fingerprint).hexdigest() + fingerprint = hashlib.sha1(fingerprint).hexdigest().upper() if not fingerprint: fingerprint = '0' * 40 - return nickname + separator + fingerprint + return prefix + fingerprint + separator + nickname def _checkServerDescriptor(self, descriptor): # If we're parsing the server-descriptor, require a networkstatus

1 0

[bridgedb/master] Refactor Bridge.updateFromExtraInfoDescriptor() transport handling code.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit ff1349df3b04225ce62358b39ad2f94cce039f27 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 24 08:58:54 2014 +0000 Refactor Bridge.updateFromExtraInfoDescriptor() transport handling code. --- lib/bridgedb/bridges.py | 91 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 29 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 71afe3b..4327f28 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -1449,6 +1449,21 @@ class Bridge(BridgeBackwardsCompatibility): def updateFromExtraInfoDescriptor(self, descriptor, verify=True): """Update this bridge's information from an extrainfo descriptor. + Stem's + :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor` + parses extrainfo ``transport`` lines into a dictionary with the + following structure:: + + {u'obfs2': (u'34.230.223.87', 37339, []), + u'obfs3': (u'34.230.223.87', 37338, []), + u'obfs4': (u'34.230.223.87', 37341, [ + (u'iat-mode=0,' + u'node-id=2a79f14120945873482b7823caabe2fcde848722,' + u'public-key=0a5b046d07f6f971b7776de682f57c5b9cdc8fa060db7ef59de82e721c8098f4')]), + u'scramblesuit': (u'34.230.223.87', 37340, [ + u'password=ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'])} + + .. todo:: The ``transport`` attribute of Stem's ``BridgeExtraInfoDescriptor`` class is a dictionary that uses the Pluggable Transport's eype as the keys. Meaning that if a bridge @@ -1473,36 +1488,54 @@ class Bridge(BridgeBackwardsCompatibility): self.descriptors['extrainfo'] = descriptor - updatedTransports = [] + oldTransports = self.transports[:] + + for methodname, (address, port, args) in descriptor.transport.items(): + updated = False + # See if we already know about this transport. If so, update its + # info; otherwise, add a new transport below. + for pt in self.transports: + if pt.methodname == methodname: + + logging.info("Found old %s transport for %s... Updating..." + % (methodname, self)) + + if not (address == str(pt.address)) and (port == pt.port): + logging.info(("Address/port for %s transport for " + "%s changed: old=%s:%s new=%s:%s") + % (methodname, self, pt.address, pt.port, + address, port)) + + oldTransports.remove(pt) + pt.updateFromStemTransport(str(self.fingerprint), + methodname, + (address, port, args,)) + updated = True + break + + if updated: + continue + else: + # We didn't update it. It must be a new transport for this + # bridges that we're hearing about for the first time, so add + # it: + logging.info( + "Received new %s pluggable transport for bridge %s." + % (methodname, safelog.logSafely(self.fingerprint))) + transport = PluggableTransport() + transport.updateFromStemTransport(str(self.fingerprint), + methodname, + (address, port, args,)) + self.transports.append(transport) + + # These are the pluggable transports which we knew about before, which + # however were not updated in this descriptor, ergo the bridge must + # not have them any more: + for pt in oldTransports: + logging.info("Removing dead transport for bridge %s: %s %s:%s %s" % + (self, pt.methodname, pt.address, pt.port, pt.arguments)) + self.transports.remove(pt) - for transport in self.transports: - for methodname, kitchenSink in descriptor.transport: - if transport.methodname == methodname: - if transport.address == kitchenSink[0]: - logging.info( - "Updating %s pluggable transport for bridge %s." % - (methodname, safelog.logSafely(self.fingerprint))) - updatedTransports.append(transport) - transport.updateFromStemTransport(self.fingerprint, - methodname, - kitchenSink) - else: - logging.info( - ("Received new %s pluggable transport for bridge " - "%s.") % - (methodname, safelog.logSafely(self.fingerprint))) - pt = PluggableTransport() - pt.updateFromStemTransport(self.fingerprint, - methodname, - kitchenSink) - updatedTransports.append(pt) - self.transports.append(pt) - - dead = set(self.transports).difference(set(updatedTransports)) - logging.info("The following transports for bridge %s died: %s" - % (self, ' '.join(dead))) - for died in dead: - self.transports.remove(died) # Bridge Stability (`#5482 <https://bugs.torproject.org>`_) properties. @property

1 0

[bridgedb/master] Stem apparently sets port 0 to 'anyport'.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit b476afe9422903207ceed4ac358351fdb9382712 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 24 08:55:40 2014 +0000 Stem apparently sets port 0 to 'anyport'. --- lib/bridgedb/bridges.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 735c603..f612565 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -468,7 +468,12 @@ class PluggableTransport(BridgeAddressBase): self.fingerprint = str(fingerprint) self.methodname = str(methodname) self.address = kitchenSink[0] - self.port = int(kitchenSink[1]) + + port = kitchenSink[1] + if port == 'anyport': # IDK. Stem, WTF? + port = 0 + + self.port = int(port) self.arguments = self._parseArgumentsIntoDict(kitchenSink[2]) self._runChecks()

1 0

[bridgedb/master] Move Main.updateBridgeHistory() into separate function.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 3f9e104bd5396ef1e264161b7b3c828505eec992 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 24 09:14:02 2014 +0000 Move Main.updateBridgeHistory() into separate function. --- lib/bridgedb/Main.py | 62 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index e1963f5..47a869c 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -34,12 +34,50 @@ from bridgedb.configure import loadConfig from bridgedb.configure import Conf from bridgedb.parse import options from bridgedb.parse.addr import isIPAddress +from bridgedb.schedule import toUnixSeconds import bridgedb.Bridges as Bridges import bridgedb.Dist as Dist import bridgedb.Storage +def updateBridgeHistory(bridges, timestamps): + """Process all the timestamps and update the bridge stability statistics in + the database. + + .. warning: This function is extremely expensive, and will keep getting + more and more expensive, on a linearithmic scale, every time it is + called. Blame the :mod:`bridgedb.Stability` module. + + :param dict bridges: All bridges from the descriptors, parsed into + :class:`bridgedb.bridges.Bridge`s. + :param dict timestamps: A dictionary whose keys are bridge fingerprints, + and whose values are lists of integers, each integer being a timestamp + (in seconds since Unix Epoch) for when a descriptor for that bridge + was published. + :rtype: dict + :returns: The original **timestamps**, but which each list of integers + (re)sorted. + """ + logging.debug("Beginning bridge stability calculations") + sortedTimestamps = {} + + for fingerprint, stamps in timestamps.items()[:]: + stamps.sort() + bridge = bridges[fingerprint] + for timestamp in stamps: + logging.debug( + ("Adding/updating timestamps in BridgeHistory for %s in " + "database: %s") % (fingerprint, timestamp)) + timestamp = toUnixSeconds(timestamp.timetuple()) + bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) + # Replace the timestamps so the next sort is (hopefully) less + # expensive: + sortedTimestamps[fingerprint] = stamps + + logging.debug("Stability calculations complete") + return sortedTimestamps + def load(state, splitter, clear=False): """Read and parse all descriptors, and load into a bridge splitter. @@ -168,30 +206,10 @@ def load(state, splitter, clear=False): logging.debug("Closing blocking-countries document") f.close() - def updateBridgeHistory(bridges, timestamps): - if not hasattr(state, 'config'): - logging.info("updateBridgeHistory(): Config file not set "\ - "in State file.") - return - if state.COLLECT_TIMESTAMPS: - logging.debug("Beginning bridge stability calculations") - for bridge in bridges.values(): - if bridge.getID() in timestamps.keys(): - ts = timestamps[bridge.getID()][:] - ts.sort() - for timestamp in ts: - logging.debug( - "Updating BridgeHistory timestamps for %s: %s" - % (bridge.fingerprint, timestamp)) - bridgedb.Stability.addOrUpdateBridgeHistory( - bridge, timestamp) - logging.debug("Stability calculations complete") - - reactor.callInThread(updateBridgeHistory, bridges, timestamps) + if state.COLLECT_TIMESTAMPS: + reactor.callInThread(updateBridgeHistory, bridges, timestamps) - bridges = None state.save() - return def _reloadFn(*args): """Placeholder callback function for :func:`_handleSIGHUP`."""

1 0

[bridgedb/master] Fix method parameters for bridges.Bridge.nickname del().
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit e27be293650207a36ffc93746ad88614407462f3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Dec 28 12:26:53 2014 +0000 Fix method parameters for bridges.Bridge.nickname del(). * FIXES a bug where the del() method for the nickname property of a Bridge wanted a value, when del() should not need parameters (other than self). --- lib/bridgedb/bridges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 5aa801e..f1bf6dc 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -512,7 +512,7 @@ class BridgeBase(BridgeAddressBase): self._nickname = value @nickname.deleter - def nickname(self, value): + def nickname(self): """Reset this Bridge's nickname.""" self._nickname = None

1 0

[bridgedb/master] Add more logging to BridgeSplitter distributor placement.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit be732f4a7461a0a1a41ee90b4ed86d6ac6534baf Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 24 09:10:02 2014 +0000 Add more logging to BridgeSplitter distributor placement. --- lib/bridgedb/Bridges.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 43edc3d..b6c51d4 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -685,6 +685,8 @@ class BridgeSplitter(BridgeHolder): pos = bisect.bisect_right(self.pValues, n) - 1 assert 0 <= pos < len(self.rings) ringname = self.rings[pos] + logging.info("%s placing bridge %s into hashring %s (via n=%s, pos=%s)." + % (self.__class__.__name__, bridge, ringname, n, pos)) validRings = self.rings + self.pseudoRings @@ -829,6 +831,9 @@ class FilteredBridgeSplitter(BridgeHolder): # I suppose since it contains memory addresses, it *is* technically # likely to be a unique string, but it is messy. + logging.debug("Adding '%s' subring to %s..." + % (ringname, self.__class__.__name__)) + if not isinstance(subring, BridgeHolder): logging.fatal("%s hashring can't add invalid subring: %r" % (self.distributorName, subring))

1 0

[bridgedb/master] Add comment to note that Stem also checks nickname spec conformity.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 842a1e26a15b83eb563671bb0929fe7a7e0f2e6e Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Dec 28 12:26:06 2014 +0000 Add comment to note that Stem also checks nickname spec conformity. --- lib/bridgedb/bridges.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 4327f28..5aa801e 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -503,10 +503,13 @@ class BridgeBase(BridgeAddressBase): def nickname(self, value): """Set this Bridge's nickname to **value**. + .. note:: We don't need to call + :func:`bridgedb.parse.nickname.isValidRouterNickname() since Stem + will check nickname specification conformity. + :param str value: The nickname of this Bridge. """ - if isValidRouterNickname(value): - self._nickname = value + self._nickname = value @nickname.deleter def nickname(self, value):

1 0

[bridgedb/master] Fix a bridge.fingerprint to be uppercased in test_bridges.py.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 61f6b4f5c89a78b9cf6268a30530588bb4a33323 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 24 09:15:50 2014 +0000 Fix a bridge.fingerprint to be uppercased in test_bridges.py. --- lib/bridgedb/test/test_bridges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index 2805c6d..cca920d 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -67,7 +67,7 @@ class BridgeIntegrationTests(unittest.TestCase): self.nickname = 'unnamed' self.ip = ipaddr.IPAddress('127.0.0.1') self.orport = '9001' - self.fingerprint = 'a1cc8dfef1fa11af9c40af1054df9daf45250556' + self.fingerprint = 'A1CC8DFEF1FA11AF9C40AF1054DF9DAF45250556' self.id_digest = a2b_hex(self.fingerprint) self.or_addresses = {ipaddr.IPAddress('6.6.6.6'): PortList(6666), ipaddr.IPAddress('42.1.42.1'): PortList(443)}

1 0

[bridgedb/master] Refactor Main.load() to use new Stem-based parsers.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 2fb7f89751bd853fc0ff1c30f752064c12dc852b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 24 09:14:49 2014 +0000 Refactor Main.load() to use new Stem-based parsers. --- lib/bridgedb/Main.py | 156 ++++++++++++++++++++++++-------------------------- 1 file changed, 76 insertions(+), 80 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 47a869c..7bc5cff 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -28,10 +28,16 @@ from bridgedb import safelog from bridgedb import schedule from bridgedb import util from bridgedb.bridges import InvalidPluggableTransportIP +from bridgedb.bridges import MalformedBridgeInfo from bridgedb.bridges import MalformedPluggableTransport +from bridgedb.bridges import MissingServerDescriptorDigest +from bridgedb.bridges import ServerDescriptorDigestMismatch +from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus from bridgedb.bridges import PluggableTransport +from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig from bridgedb.configure import Conf +from bridgedb.parse import descriptors from bridgedb.parse import options from bridgedb.parse.addr import isIPAddress from bridgedb.schedule import toUnixSeconds @@ -103,91 +109,72 @@ def load(state, splitter, clear=False): logging.info("Loading bridges...") bridges = {} - status = {} - addresses = {} timestamps = {} - bridges = {} - desc_digests = {} - ei_digests = {} - - logging.info("Opening network status file: %s" % state.STATUS_FILE) - f = open(state.STATUS_FILE, 'r') - for (ID, nickname, desc_digest, running, stable, - ORaddr, ORport, or_addresses, - timestamp) in Bridges.parseStatusFile(f): - bridge = Bridges.Bridge(nickname, ORaddr, ORport, id_digest=ID, - or_addresses=or_addresses) - bridge.assertOK() - bridge.setStatus(running, stable) - bridge.setDescriptorDigest(desc_digest) - bridges[ID] = bridge - - if ID in timestamps.keys(): - timestamps[ID].append(timestamp) + + logging.info("Opening networkstatus file: %s" % state.STATUS_FILE) + networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE) + logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE) + + logging.info("Processing networkstatus descriptors...") + for router in networkstatuses: + bridge = Bridge() + bridge.updateFromNetworkStatus(router) + bridge.flags.update(router.flags) + + try: + bridge.assertOK() + except MalformedBridgeInfo as error: + logging.warn(str(error)) else: - timestamps[ID] = [timestamp] - logging.debug("Closing network status file") - f.close() - - for fname in state.BRIDGE_FILES: - logging.info("Opening bridge-server-descriptor file: '%s'" % fname) - f = open(fname, 'r') - desc_digests.update(Bridges.getDescriptorDigests(f)) - if state.COLLECT_TIMESTAMPS: - for bridge in bridges.values(): - if bridge.getID() in timestamps.keys(): - ts = timestamps[bridge.getID()][:] - ts.sort() - for timestamp in ts: - logging.debug( - "Adding/updating timestamps in BridgeHistory for "\ - "'%s' in database: %s" - % (bridge.fingerprint, timestamp)) - bridgedb.Stability.addOrUpdateBridgeHistory( - bridge, timestamp) - logging.debug("Closing bridge-server-descriptor file: '%s'" % fname) - f.close() + bridges[bridge.fingerprint] = bridge - for ID in bridges.keys(): - bridge = bridges[ID] - if bridge.desc_digest in desc_digests: - bridge.setVerified() - bridge.setExtraInfoDigest(desc_digests[bridge.desc_digest]) - # We attempt to insert all bridges. If the bridge is not - # running, then it is skipped during the insertion process. - splitter.insert(bridge) + for filename in state.BRIDGE_FILES: + logging.info("Opening bridge-server-descriptor file: '%s'" % filename) + serverdescriptors = descriptors.parseServerDescriptorsFile(filename) + logging.debug("Closing bridge-server-descriptor file: '%s'" % filename) - # read pluggable transports from extra-info document - # XXX: should read from networkstatus after bridge-authority - # does a reachability test - for filename in state.EXTRA_INFO_FILES: - logging.info("Opening extra-info file: '%s'" % filename) - f = open(filename, 'r') - for transport in Bridges.parseExtraInfoFile(f): - ID, method_name, address, port, argdict = transport + for router in serverdescriptors: try: - if bridges[ID].running: - logging.info("Adding %s transport to running bridge" - % method_name) - try: - bridgePT = PluggableTransport(Bridges.toHex(ID), - method_name, address, - port, argdict) - except (InvalidPluggableTransportIP, - MalformedPluggableTransport) as error: - logging.warn(error) - else: - bridges[ID].transports.append(bridgePT) - if not bridgePT in bridges[ID].transports: - logging.critical( - "Added a transport, but it disappeared!", - "\tTransport: %r" % bridgePT) - except KeyError as error: - logging.error("Could not find bridge with fingerprint '%s'." - % Bridges.toHex(ID)) - logging.debug("Closing extra-info file: '%s'" % filename) - f.close() - + bridges[router.fingerprint].updateFromServerDescriptor(router) + except KeyError: + logging.warn( + ("Received server descriptor for bridge '%s' which wasn't " + "in the networkstatus!") % router.fingerprint) + continue + except (ServerDescriptorWithoutNetworkstatus, + MissingServerDescriptorDigest, + ServerDescriptorDigestMismatch) as error: + logging.warn(str(error)) + # Reject any routers whose server descriptors didn't pass + # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those + # bridges who don't have corresponding networkstatus + # documents, or whose server descriptor digests don't check + # out: + bridges.pop(router.fingerprint) + continue + + if state.COLLECT_TIMESTAMPS: + # Update timestamps from server descriptors, not from network + # status descriptors (because networkstatus documents and + # descriptors aren't authenticated in any way): + if bridge.fingerprint in timestamps.keys(): + timestamps[bridge.fingerprint].append(router.published) + else: + timestamps[bridge.fingerprint] = [router.published] + + extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES) + for fingerprint, router in extrainfos.items(): + try: + bridges[fingerprint].updateFromExtraInfoDescriptor(router) + except MalformedBridgeInfo as error: + logging.warn(str(error)) + except KeyError as error: + logging.warn(("Received extrainfo descriptor for bridge '%s', " + "but could not find bridge with that fingerprint.") + % router.fingerprint) + + # XXX TODO refactor the next block according with new parsers for OONI + # bridge-reachability reports: if state.COUNTRY_BLOCK_FILE: logging.info("Opening Blocking Countries file %s" % state.COUNTRY_BLOCK_FILE) @@ -206,6 +193,15 @@ def load(state, splitter, clear=False): logging.debug("Closing blocking-countries document") f.close() + inserted = 0 + logging.info("Inserting %d bridges into splitter..." % len(bridges)) + for fingerprint, bridge in bridges.items(): + # We attempt to insert all bridges. If the bridge is not running, then + # it is skipped during the insertion process. + splitter.insert(bridge) + inserted += 1 + logging.info("Done inserting %d bridges into splitter." % inserted) + if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps)

1 0