February 2015 - tor-commits - lists.torproject.org

[bridgedb/develop] Add more info to docstring for `b.p.descriptors.parseBridgeExtraInfoFiles()`.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 1e4de24ac711dbcbe6b1992da951a90ee64c2e8e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 04:00:27 2014 +0000 Add more info to docstring for `b.p.descriptors.parseBridgeExtraInfoFiles()`. --- lib/bridgedb/parse/descriptors.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 939ecb0..ab0d37f 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/… [View More]

1 0

[bridgedb/develop] Fix faulty unittests which expected parsed extrainfos to have an `address`.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 41f0f5c255740d777764fa7a336769c317b0cc26 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 03:57:35 2014 +0000 Fix faulty unittests which expected parsed extrainfos to have an `address`. --- lib/bridgedb/test/test_parse_descriptors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 85863e4..2a1171e 100644 --- a/lib/bridgedb/test/… [View More]

1 0

[bridgedb/develop] Rewrite `b.p.descriptors.deduplicate()` to not modify while indexing.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit fe70415269693948bdfc5c8ea3abfab2b1d86c49 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 26 04:02:05 2014 +0000 Rewrite `b.p.descriptors.deduplicate()` to not modify while indexing. --- lib/bridgedb/parse/descriptors.py | 70 +++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index ab0d37f..a0806e2 100644 --- a/lib/bridgedb/parse/… [View More]descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -98,45 +98,61 @@ def deduplicate(descriptors): :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`s, :api:`stem.descriptor.router_status_entry.RouterStatusEntryV2`s. """ - duplicates = [] - nonDuplicates = [] + duplicates = {} + nonDuplicates = {} for descriptor in descriptors: - router = descriptors.pop(descriptors.index(descriptor)) - fingerprint = router.fingerprint + fingerprint = descriptor.fingerprint logging.debug("Deduplicating %s descriptor for router %s" - % (str(router.__class__).rsplit('.', 1)[1], + % (str(descriptor.__class__).rsplit('.', 1)[1], safelog.logSafely(fingerprint))) - for possibleDuplicate in descriptors: - if fingerprint == possibleDuplicate.fingerprint: - logging.warn("Duplicate extra-info descriptor for router %s" - % safelog.logSafely(fingerprint)) - if router.published > possibleDuplicate.published: - # The router is newer than the duplicate, so get rid of - # the duplicate: - duplicates.append(possibleDuplicate) - elif router.published < possibleDuplicate.published: - # The router is older than the duplicate, so replace our - # router: - duplicates.append(router) - router = possibleDuplicate - else: - duplicates.append(possibleDuplicate) - logging.warn(("Duplicate descriptor and original " - "descriptor for router %s both had the same " - "timestamp: %s") - % (safelog.logSafely(fingerprint), - router.published)) + if fingerprint in nonDuplicates.keys(): + # We already found a descriptor for this fingerprint: + conflict = nonDuplicates[fingerprint] + + # If the descriptor we are currently parsing is newer than the + # last one we found: + if descriptor.published > conflict.published: + # And if this is the first duplicate we've found for this + # router, then create a list in the ``duplicates`` dictionary + # for the router: + if not fingerprint in duplicates.keys(): + duplicates[fingerprint] = list() + # Add this to the list of duplicates for this router: + duplicates[fingerprint].append(conflict) + # Finally, put the newest descriptor in the ``nonDuplicates`` + # dictionary: + nonDuplicates[fingerprint] = descriptor + # Same thing, but this time the one we're parsing is older: + elif descriptor.published < conflict.published: + if not fingerprint in duplicates.keys(): + duplicates[fingerprint] = list() + duplicates[fingerprint].append(descriptor) + # This *shouldn't* happen. It would mean that two descriptors for + # the same router had the same timestamps, probably meaning there + # is a severely-messed up OR implementation out there. Let's log + # its fingerprint (no matter what!) so that we can look up its + # ``platform`` line in its server-descriptor and tell whoever + # wrote that code that they're probably (D)DOSing the Tor network. else: - nonDuplicates.append(router) + logging.warn(("Duplicate descriptor with identical timestamp " + "(%s) for router with fingerprint '%s'!") + % (descriptor.published, fingerprint)) + + # Hoorah! No duplicates! (yet...) + else: + nonDuplicates[fingerprint] = descriptor logging.info("Descriptor deduplication finished.") logging.info("Number of duplicates: %d" % len(duplicates)) + for (fingerprint, dittos) in duplicates.items(): + logging.info(" For %s: %d duplicates" + % (safelog.logSafely(fingerprint), len(dittos))) logging.info("Number of non-duplicates: %d" % len(nonDuplicates)) - return nonDuplicates + return nonDuplicates def parseBridgeExtraInfoFiles(*filenames, **kwargs): """Parse files which contain ``@type bridge-extrainfo-descriptor``s. [View Less]

1 0

[bridgedb/develop] Add error handling for cases when Stem cannot parse an extrainfo file.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 1f19568e61142521ddaccbc92a461ce831ecb255 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 02:35:51 2014 +0000 Add error handling for cases when Stem cannot parse an extrainfo file. If a file containing bridge-extrainfo descriptors cannot be parsed the first time, and validation was enabled, then we parse it again with Stem's validation functionality disabled. If we still can't parse the file, we save a timestamped copy of the file for later … [View More]debugging an continue with the next file. --- lib/bridgedb/parse/descriptors.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 63d9bf2..6eac164 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -238,10 +238,38 @@ def parseBridgeExtraInfoFiles(*filenames, **kwargs): validate = False for filename in filenames: + document = None + documentWasUnparseable = False + logging.info("Parsing %s descriptors with Stem: %s" % (descriptorType, filename)) - document = parse_file(filename, descriptorType, validate=validate) - descriptors.extend([router for router in document]) + try: + document = parse_file(filename, descriptorType, validate=validate) + except ValueError as error: + documentWasUnparseable = True + + if documentWasUnparseable: + logging.warn(("Stem ran into an exception while parsing extrainfo " + "file '%s'!") % filename) + logging.debug("Error while parsing extrainfo file:\n%s" + % str(error)) + + if documentWasUnparseable and (validate is True): + logging.info(("Retrying parsing of extrainfo file '%s' with " + "validation disabled...") % filename) + try: + document = parse_file(filename, descriptorType, validate=False) + except ValueError as another: + logging.critical(("We were still unable to parse extrainfo " + "file on the second attempt! Bailing!")) + else: + documentWasUnparseable = False + + if documentWasUnparseable: + _copyUnparseableDescriptorFile(filename) + + if document: + descriptors.extend([router for router in document]) routers = deduplicate(descriptors) return routers [View Less]

1 0

[bridgedb/develop] Fix unittests for return values of parseNetworkstatusFile().
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit c2f79261dfcc9e1f08006177e5881ca189cf6270 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Aug 29 07:36:41 2014 +0000 Fix unittests for return values of parseNetworkstatusFile(). `bridgedb.parse.descriptors.parseNetworkstatusFile()` now returns list of `stem.descriptors.router_status_entry.RouterStatusEntryV3s`. --- lib/bridgedb/test/test_parse_descriptors.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/… [View More]bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index b947a39..1abd08f 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -22,7 +22,7 @@ HAS_STEM = False try: from stem.descriptor.server_descriptor import RelayDescriptor from stem.descriptor.extrainfo_descriptor import BridgeExtraInfoDescriptor - from stem.descriptor.router_status_entry import RouterStatusEntryV2 + from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb.parse import descriptors except (ImportError, NameError), error: print("There was an error importing stem: %s" % error) @@ -185,7 +185,7 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) - self.assertIsInstance(routers, dict) + self.assertIsInstance(routers, list) def test_parse_descriptors_parseBridgeNetworkStatusFile_has_RouterStatusEntryV2(self): """The items in the dict returned from @@ -197,8 +197,8 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) - fingerprint, bridge = routers.items()[0] - self.assertIsInstance(bridge, RouterStatusEntryV2) + bridge = routers[0] + self.assertIsInstance(bridge, RouterStatusEntryV3) def test_parse_descriptors_parseBridgeNetworkStatusFile_1(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with one bridge @@ -209,9 +209,9 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) - fingerprint, bridge = routers.items()[0] + bridge = routers[0] self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(fingerprint, + self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') def test_parse_descriptors_parseBridgeNetworkStatusFile_2(self): @@ -224,9 +224,9 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_0, BRIDGE_NETWORKSTATUS_1) routers = descriptors.parseNetworkStatusFile(descFile) - fingerprint, bridge = routers.items()[0] + bridge = routers[0] self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(fingerprint, + self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') def test_parse_descriptors_parseBridgeExtraInfoFiles_return_type(self): [View Less]

1 0

[bridgedb/develop] Change `skipHeaders` to `skipAnnotations` in b.p.d.parseNetworkStatusFile().
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 95f6367e0e39b6ed0e43ee8eb33ced069f24e9ce Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 3 06:32:31 2014 +0000 Change `skipHeaders` to `skipAnnotations` in b.p.d.parseNetworkStatusFile(). In order to match the terminology used in Tor's source code. --- lib/bridgedb/parse/descriptors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 49d6e8a..6c6fd3e … [View More]

1 0

[bridgedb/develop] Bump Leekspin to version 1.1.4.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit e16f2638a92b2c426d84b3d8cc9c20d4aca04b82 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Sep 7 00:36:29 2014 +0000 Bump Leekspin to version 1.1.4. I changed the `leekspin.util.randomIP()` function to only generate "valid" addresses, which prevents us from hitting MalformedPluggableTransport errors more than we normally would with real bridge descriptors. --- .test.requirements.txt | 2 +- .travis.requirements.txt | 2 +- setup.py … [View More]

1 0

[bridgedb/develop] Deprecate old bridgedb.Bridges.Bridge class.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 07f01e9df28b35433ca7b6891855ab64c3fa402e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 4 22:23:42 2014 +0000 Deprecate old bridgedb.Bridges.Bridge class. --- lib/bridgedb/Bridges.py | 252 -------------------------------------- lib/bridgedb/test/deprecated.py | 256 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 252 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index dead9f2..36637aa 100644 … [View More]--- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -75,258 +75,6 @@ def is_valid_ip(ip): return True -class Bridge(object): - """Holds information for a single bridge, along with any Pluggable - Transports it is also running. - - :attr str nickname: The bridge's nickname. Not currently used. - :attr ip: (:class:`ipaddr.IPAddress`) The bridge's IPv4 address, specified - on the 'r'-line in a networkstatus document. - :attr int orport: The bridge's OR port. - :attr dict or_addresses: The bridges alternate IP addresses. The keys - should be instances of ``ipaddr.IPAddress``, and the value should be a - :class:`bridgedb.parse.addr.PortList` for the port(s) on which that - address is listening. - :attr list transports: List of :class:`PluggableTransport` instances for - each PT which the bridge supports. - :attr str fingerprint: The bridge's identity digest, in lowercase hex, - without whitespace. - :attr bool running: ``True``, if this bridge was given the ``Running`` flag. - :attr bool stable: ``True``, if this bridge was given the ``Stable`` flag. - :attr dict blockingCountries: A dictionary whose keys are strings of - ``"IP:port"`` pairs, and the keys are lists of two letter country - codes which block that IP:port. For example:: - {"1.2.3.4:9001": ['sk', 'us', 'ir', 'cn']} - :attr str desc_digest: SHA1 hexdigest of the bridge's descriptor as - defined in the networkstatus document. - :attr str ei_digest: SHA1 hexdigest of the bridge's extra-info document as - given in the bridge's descriptor, corresponding to desc_digest. - :attr bool verified: Did we receive the descriptor for this bridge that - was specified in the networkstatus? - """ - def __init__(self, nickname, ip, orport, fingerprint=None, id_digest=None, - or_addresses=None, transports=None): - """Create a new Bridge. One of fingerprint and id_digest must be set. - """ - self.nickname = nickname - self.ip = ip - self.orport = orport - if not or_addresses: or_addresses = {} - self.or_addresses = or_addresses - if not transports: transports = [] - self.transports = transports - self.running = self.stable = None - self.blockingCountries = {} - self.desc_digest = None - self.ei_digest = None - self.verified = False - - if id_digest is not None: - assert fingerprint is None - if len(id_digest) != DIGEST_LEN: - raise TypeError("Bridge with invalid ID") - self.fingerprint = toHex(id_digest) - elif fingerprint is not None: - if not isValidFingerprint(fingerprint): - raise TypeError("Bridge with invalid fingerprint (%r)"% - fingerprint) - self.fingerprint = fingerprint.lower() - else: - raise TypeError("Bridge with no ID") - - def setDescriptorDigest(self, digest): - """Set the descriptor digest, specified in the NS.""" - self.desc_digest = digest - - def setExtraInfoDigest(self, digest): - """Set the extra-info digest, specified in the descriptor.""" - self.ei_digest = digest - - def setVerified(self): - """Call when the bridge's descriptor is parsed""" - self.verified = True - - def isVerified(self): - """Returns the truthiness of ``verified``""" - return self.verified - - def getID(self): - """Return the bridge's identity digest.""" - return fromHex(self.fingerprint) - - def __repr__(self): - """Return a piece of python that evaluates to this bridge.""" - if self.or_addresses: - return "Bridge(%r,%r,%d,%r,or_addresses=%s)"%( - self.nickname, self.ip, self.orport, self.fingerprint, - self.or_addresses) - return "Bridge(%r,%r,%d,%r)"%( - self.nickname, self.ip, self.orport, self.fingerprint) - - def getConfigLine(self, includeFingerprint=False, addressClass=None, - request=None, transport=None): - """Returns a valid bridge line for inclusion in a torrc. - - :param bool includeFingerprint: If ``True``, include the - ``fingerprint`` of this :class:`Bridge` in the returned bridge - line. - :param DOCDOC addressClass: Type of address to choose. - :param str request: A string unique to this request e.g. email-address - or ``uniformMap(ip)`` or ``'default'``. - :param str transport: A pluggable transport method name. - """ - - if not request: request = 'default' - digest = getHMACFunc('Order-Or-Addresses')(request) - pos = long(digest[:8], 16) # lower 8 bytes -> long - - # default address type - if not addressClass: addressClass = ipaddr.IPv4Address - - # pluggable transports - if transport: - # filter by 'methodname' - transports = filter(lambda x: transport == x.methodname, - self.transports) - # filter by 'addressClass' - transports = filter(lambda x: isinstance(x.address, addressClass), - transports) - if transports: - pt = transports[pos % len(transports)] - return pt.getTransportLine(includeFingerprint) - - # filter addresses by address class - addresses = filter(lambda x: isinstance(x[0], addressClass), - self.or_addresses.items()) - - # default ip, orport should get a chance at being selected - if isinstance(self.ip, addressClass): - addresses.insert(0,(self.ip, addr.PortList(self.orport))) - - if addresses: - address,portlist = addresses[pos % len(addresses)] - if isinstance(address, ipaddr.IPv6Address): ip = "[%s]"%address - else: ip = "%s"%address - orport = portlist[pos % len(portlist)] - - if includeFingerprint: - return "%s:%d %s" % (ip, orport, self.fingerprint) - else: - return "%s:%d" % (ip, orport) - - def getAllConfigLines(self,includeFingerprint=False): - """Generator. Iterate over all valid config lines for this bridge.""" - for address,portlist in self.or_addresses.items(): - if type(address) is ipaddr.IPv6Address: - ip = "[%s]" % address - else: - ip = "%s" % address - - for orport in portlist: - if includeFingerprint: - yield "bridge %s:%d %s" % (ip,orport,self.fingerprint) - else: - yield "bridge %s:%d" % (ip,orport) - for pt in self.transports: - yield pt.getTransportLine(includeFingerprints) - - - def assertOK(self): - assert is_valid_ip(self.ip) - assert isValidFingerprint(self.fingerprint) - assert 1 <= self.orport <= 65535 - if self.or_addresses: - for address, portlist in self.or_addresses.items(): - assert is_valid_ip(address) - for port in portlist: - assert type(port) is int - assert 1 <= port <= 65535 - - def setStatus(self, running=None, stable=None): - if running is not None: - self.running = running - if stable is not None: - self.stable = stable - - def isBlocked(self, countryCode, addressClass, methodname=None): - """ if at least one address:port of the selected addressClass and - (optional) transport type is not blocked in countryCode, return True - """ - # 1) transport is specified - if methodname is not None: - for transport in self.transports: - key = "%s:%s" % (transport.address, transport.port) - if (isinstance(transport.address, addressClass) - and transport.methodname.lower() == methodname.lower()): - try: - if countryCode not in self.blockingCountries[key]: - return False - except KeyError: - return False # no blocklist - return True - # 2) no transport specified (default) - else: - # 3) check primary ip, port - # XXX: could be more elegant if ip,orport were not special case - if isinstance(self.ip, addressClass): - key = "%s:%s" % (self.ip, self.orport) - try: - if countryCode not in self.blockingCountries[key]: - return False - except KeyError: return False # no blocklist - - # 4) check or addresses - for address,portlist in self.or_addresses.items(): - if isinstance(address, addressClass): - # check each port - for port in portlist: - key = "%s:%s" % (address, port) - try: - if countryCode not in self.blockingCountries[key]: - return False - except KeyError: return False # no blocklist - return True - - # Bridge Stability (#5482) properties. - @property - def familiar(self): - """ - A bridge is 'familiar' if 1/8 of all active bridges have appeared - more recently than it, or if it has been around for a Weighted Time of 8 days. - """ - with bridgedb.Storage.getDB() as db: - return db.getBridgeHistory(self.fingerprint).familiar - - @property - def wfu(self): - """Weighted Fractional Uptime""" - with bridgedb.Storage.getDB() as db: - return db.getBridgeHistory(self.fingerprint).weightedFractionalUptime - - @property - def weightedTime(self): - """Weighted Time""" - with bridgedb.Storage.getDB() as db: - return db.getBridgeHistory(self.fingerprint).weightedTime - - @property - def wmtbac(self): - """Weighted Mean Time Between Address Change""" - with bridgedb.Storage.getDB() as db: - return db.getBridgeHistory(self.fingerprint).wmtbac - - @property - def tosa(self): - """the Time On Same Address (TOSA)""" - with bridgedb.Storage.getDB() as db: - return db.getBridgeHistory(self.fingerprint).tosa - - @property - def weightedUptime(self): - """Weighted Uptime""" - with bridgedb.Storage.getDB() as db: - return db.getBridgeHistory(self.fingerprint).weightedUptime - def getDescriptorDigests(desc): """Return the SHA-1 hash hexdigests of all descriptor descs diff --git a/lib/bridgedb/test/deprecated.py b/lib/bridgedb/test/deprecated.py index 40cfe0a..d85f6f9 100644 --- a/lib/bridgedb/test/deprecated.py +++ b/lib/bridgedb/test/deprecated.py @@ -124,6 +124,262 @@ class ParseORAddressError(Exception): @deprecate.deprecated( Version('bridgedb', 0, 2, 4), + replacement='bridgedb.bridges.Bridge') +class Bridge(object): + """Holds information for a single bridge, along with any Pluggable + Transports it is also running. + + :attr str nickname: The bridge's nickname. Not currently used. + :attr ip: (:class:`ipaddr.IPAddress`) The bridge's IPv4 address, specified + on the 'r'-line in a networkstatus document. + :attr int orport: The bridge's OR port. + :attr dict or_addresses: The bridges alternate IP addresses. The keys + should be instances of ``ipaddr.IPAddress``, and the value should be a + :class:`bridgedb.parse.addr.PortList` for the port(s) on which that + address is listening. + :attr list transports: List of :class:`PluggableTransport` instances for + each PT which the bridge supports. + :attr str fingerprint: The bridge's identity digest, in lowercase hex, + without whitespace. + :attr bool running: ``True``, if this bridge was given the ``Running`` flag. + :attr bool stable: ``True``, if this bridge was given the ``Stable`` flag. + :attr dict blockingCountries: A dictionary whose keys are strings of + ``"IP:port"`` pairs, and the keys are lists of two letter country + codes which block that IP:port. For example:: + {"1.2.3.4:9001": ['sk', 'us', 'ir', 'cn']} + :attr str desc_digest: SHA1 hexdigest of the bridge's descriptor as + defined in the networkstatus document. + :attr str ei_digest: SHA1 hexdigest of the bridge's extra-info document as + given in the bridge's descriptor, corresponding to desc_digest. + :attr bool verified: Did we receive the descriptor for this bridge that + was specified in the networkstatus? + """ + def __init__(self, nickname, ip, orport, fingerprint=None, id_digest=None, + or_addresses=None, transports=None): + """Create a new Bridge. One of fingerprint and id_digest must be set. + """ + self.nickname = nickname + self.ip = ip + self.orport = orport + if not or_addresses: or_addresses = {} + self.or_addresses = or_addresses + if not transports: transports = [] + self.transports = transports + self.running = self.stable = None + self.blockingCountries = {} + self.desc_digest = None + self.ei_digest = None + self.verified = False + + if id_digest is not None: + assert fingerprint is None + if len(id_digest) != DIGEST_LEN: + raise TypeError("Bridge with invalid ID") + self.fingerprint = toHex(id_digest) + elif fingerprint is not None: + if not isValidFingerprint(fingerprint): + raise TypeError("Bridge with invalid fingerprint (%r)"% + fingerprint) + self.fingerprint = fingerprint.lower() + else: + raise TypeError("Bridge with no ID") + + def setDescriptorDigest(self, digest): + """Set the descriptor digest, specified in the NS.""" + self.desc_digest = digest + + def setExtraInfoDigest(self, digest): + """Set the extra-info digest, specified in the descriptor.""" + self.ei_digest = digest + + def setVerified(self): + """Call when the bridge's descriptor is parsed""" + self.verified = True + + def isVerified(self): + """Returns the truthiness of ``verified``""" + return self.verified + + def getID(self): + """Return the bridge's identity digest.""" + return fromHex(self.fingerprint) + + def __repr__(self): + """Return a piece of python that evaluates to this bridge.""" + if self.or_addresses: + return "Bridge(%r,%r,%d,%r,or_addresses=%s)"%( + self.nickname, self.ip, self.orport, self.fingerprint, + self.or_addresses) + return "Bridge(%r,%r,%d,%r)"%( + self.nickname, self.ip, self.orport, self.fingerprint) + + def getConfigLine(self, includeFingerprint=False, addressClass=None, + request=None, transport=None): + """Returns a valid bridge line for inclusion in a torrc. + + :param bool includeFingerprint: If ``True``, include the + ``fingerprint`` of this :class:`Bridge` in the returned bridge + line. + :param DOCDOC addressClass: Type of address to choose. + :param str request: A string unique to this request e.g. email-address + or ``uniformMap(ip)`` or ``'default'``. + :param str transport: A pluggable transport method name. + """ + + if not request: request = 'default' + digest = getHMACFunc('Order-Or-Addresses')(request) + pos = long(digest[:8], 16) # lower 8 bytes -> long + + # default address type + if not addressClass: addressClass = ipaddr.IPv4Address + + # pluggable transports + if transport: + # filter by 'methodname' + transports = filter(lambda x: transport == x.methodname, + self.transports) + # filter by 'addressClass' + transports = filter(lambda x: isinstance(x.address, addressClass), + transports) + if transports: + pt = transports[pos % len(transports)] + return pt.getTransportLine(includeFingerprint) + + # filter addresses by address class + addresses = filter(lambda x: isinstance(x[0], addressClass), + self.or_addresses.items()) + + # default ip, orport should get a chance at being selected + if isinstance(self.ip, addressClass): + addresses.insert(0,(self.ip, addr.PortList(self.orport))) + + if addresses: + address,portlist = addresses[pos % len(addresses)] + if isinstance(address, ipaddr.IPv6Address): ip = "[%s]"%address + else: ip = "%s"%address + orport = portlist[pos % len(portlist)] + + if includeFingerprint: + return "%s:%d %s" % (ip, orport, self.fingerprint) + else: + return "%s:%d" % (ip, orport) + + def getAllConfigLines(self,includeFingerprint=False): + """Generator. Iterate over all valid config lines for this bridge.""" + for address,portlist in self.or_addresses.items(): + if type(address) is ipaddr.IPv6Address: + ip = "[%s]" % address + else: + ip = "%s" % address + + for orport in portlist: + if includeFingerprint: + yield "bridge %s:%d %s" % (ip,orport,self.fingerprint) + else: + yield "bridge %s:%d" % (ip,orport) + for pt in self.transports: + yield pt.getTransportLine(includeFingerprints) + + + def assertOK(self): + assert is_valid_ip(self.ip) + assert isValidFingerprint(self.fingerprint) + assert 1 <= self.orport <= 65535 + if self.or_addresses: + for address, portlist in self.or_addresses.items(): + assert is_valid_ip(address) + for port in portlist: + assert type(port) is int + assert 1 <= port <= 65535 + + def setStatus(self, running=None, stable=None): + if running is not None: + self.running = running + if stable is not None: + self.stable = stable + + def isBlocked(self, countryCode, addressClass, methodname=None): + """ if at least one address:port of the selected addressClass and + (optional) transport type is not blocked in countryCode, return True + """ + # 1) transport is specified + if methodname is not None: + for transport in self.transports: + key = "%s:%s" % (transport.address, transport.port) + if (isinstance(transport.address, addressClass) + and transport.methodname.lower() == methodname.lower()): + try: + if countryCode not in self.blockingCountries[key]: + return False + except KeyError: + return False # no blocklist + return True + # 2) no transport specified (default) + else: + # 3) check primary ip, port + # XXX: could be more elegant if ip,orport were not special case + if isinstance(self.ip, addressClass): + key = "%s:%s" % (self.ip, self.orport) + try: + if countryCode not in self.blockingCountries[key]: + return False + except KeyError: return False # no blocklist + + # 4) check or addresses + for address,portlist in self.or_addresses.items(): + if isinstance(address, addressClass): + # check each port + for port in portlist: + key = "%s:%s" % (address, port) + try: + if countryCode not in self.blockingCountries[key]: + return False + except KeyError: return False # no blocklist + return True + + # Bridge Stability (#5482) properties. + @property + def familiar(self): + """ + A bridge is 'familiar' if 1/8 of all active bridges have appeared + more recently than it, or if it has been around for a Weighted Time of 8 days. + """ + with bridgedb.Storage.getDB() as db: + return db.getBridgeHistory(self.fingerprint).familiar + + @property + def wfu(self): + """Weighted Fractional Uptime""" + with bridgedb.Storage.getDB() as db: + return db.getBridgeHistory(self.fingerprint).weightedFractionalUptime + + @property + def weightedTime(self): + """Weighted Time""" + with bridgedb.Storage.getDB() as db: + return db.getBridgeHistory(self.fingerprint).weightedTime + + @property + def wmtbac(self): + """Weighted Mean Time Between Address Change""" + with bridgedb.Storage.getDB() as db: + return db.getBridgeHistory(self.fingerprint).wmtbac + + @property + def tosa(self): + """the Time On Same Address (TOSA)""" + with bridgedb.Storage.getDB() as db: + return db.getBridgeHistory(self.fingerprint).tosa + + @property + def weightedUptime(self): + """Weighted Uptime""" + with bridgedb.Storage.getDB() as db: + return db.getBridgeHistory(self.fingerprint).weightedUptime + + +(a)deprecate.deprecated( + Version('bridgedb', 0, 2, 4), replacement='bridgedb.bridges.PluggableTransport') class PluggableTransport(object): """A PT with reference to the parent bridge on which it is running. [View Less]

1 0

[bridgedb/develop] Workaround strange file descriptor issue in Stem.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit f59121dc25a256ae9e24f65d0e29ea4109ab102a Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Aug 29 09:58:09 2014 +0000 Workaround strange file descriptor issue in Stem. For reasons I am unable as of yet to determine, if I call `stem.descriptor.router_status_entry._parse_file()` with its `start_position` parameter, giving it the byte offset in the file to start reading at, as well as the file descriptor itself, that Stem function does *not* reliably … [View More]

1 0

[bridgedb/develop] Validate extrainfo descriptors by default.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit babbede52344e6ddf51dea2bcc61b688a7aa8c0f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 01:45:29 2014 +0000 Validate extrainfo descriptors by default. Also update the documentation of `b.p.d.parseBridgeExtraInfoFiles` to mention a method for ``router-signature`` verification, and the reasons why we use Stem's ``RelayExtraInfoDescriptor`` class, rather than the ``BridgeExtraInfoDescriptor`` class. --- lib/bridgedb/parse/descriptors.py … [View More] | 29 ++++++++++++++++++++++----- lib/bridgedb/test/test_parse_descriptors.py | 4 ++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index a7cb202..03b3f55 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -164,22 +164,41 @@ def deduplicate(descriptors): def parseBridgeExtraInfoFiles(*filenames, **kwargs): """Parse files which contain ``@type bridge-extrainfo-descriptor``s. + .. warning:: This function will *not* check that the ``router-signature`` + at the end of the extrainfo descriptor is valid. See + ``bridgedb.bridges.Bridge._verifyExtraInfoSignature`` for a method for + checking the signature. + .. note:: This function will call :func:`deduplicate` to deduplicate the extrainfo descriptors parsed from all **filenames**. :kwargs validate: If there is a ``'validate'`` keyword argument, its value will be passed along as the ``'validate'`` argument to :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`. + The ``'validate'`` keyword argument defaults to ``True``, meaning that + the hash digest stored in the ``router-digest`` line will be checked + against the actual contents of the descriptor and the extrainfo + document's signature will be verified. :rtype: dict :returns: A dictionary mapping bridge fingerprints to deduplicated - :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`s. + :api:`stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor`s. """ descriptors = [] - descriptorType = 'bridge-extra-info 1.1' - validate = False - if ('validate' in kwargs) and (kwargs['validate'] is True): - validate = True + # The ``stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`` + # class (with ``descriptorType = 'bridge-extra-info 1.1``) is unsuitable + # for our purposes for the following reasons: + # + # 1. It expects a ``router-digest`` line, which is only present in + # sanitised bridge extrainfo descriptors. + # + # 2. It doesn't check the ``router-signature`` (nor does it expect there + # to be a signature). + descriptorType = 'extra-info 1.0' + + validate = True + if ('validate' in kwargs) and (kwargs['validate'] is False): + validate = False for filename in filenames: logging.info("Parsing %s descriptors with Stem: %s" diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 953f371..c3772c7 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -21,7 +21,7 @@ HAS_STEM = False try: from stem.descriptor.server_descriptor import RelayDescriptor - from stem.descriptor.extrainfo_descriptor import BridgeExtraInfoDescriptor + from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb.parse import descriptors except (ImportError, NameError), error: @@ -252,7 +252,7 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] - self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) + self.assertIsInstance(bridge, RelayExtraInfoDescriptor) def test_parse_descriptors_parseBridgeExtraInfoFiles_one_file(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with only one [View Less]

1 0