February 2015 - tor-commits - lists.torproject.org

[bridgedb/develop] Validate extrainfo descriptors by default.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit babbede52344e6ddf51dea2bcc61b688a7aa8c0f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 01:45:29 2014 +0000 Validate extrainfo descriptors by default. Also update the documentation of `b.p.d.parseBridgeExtraInfoFiles` to mention a method for ``router-signature`` verification, and the reasons why we use Stem's ``RelayExtraInfoDescriptor`` class, rather than the ``BridgeExtraInfoDescriptor`` class. --- lib/bridgedb/parse/descriptors.py | 29 ++++++++++++++++++++++----- lib/bridgedb/test/test_parse_descriptors.py | 4 ++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index a7cb202..03b3f55 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -164,22 +164,41 @@ def deduplicate(descriptors): def parseBridgeExtraInfoFiles(*filenames, **kwargs): """Parse files which contain ``@type bridge-extrainfo-descriptor``s. + .. warning:: This function will *not* check that the ``router-signature`` + at the end of the extrainfo descriptor is valid. See + ``bridgedb.bridges.Bridge._verifyExtraInfoSignature`` for a method for + checking the signature. + .. note:: This function will call :func:`deduplicate` to deduplicate the extrainfo descriptors parsed from all **filenames**. :kwargs validate: If there is a ``'validate'`` keyword argument, its value will be passed along as the ``'validate'`` argument to :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`. + The ``'validate'`` keyword argument defaults to ``True``, meaning that + the hash digest stored in the ``router-digest`` line will be checked + against the actual contents of the descriptor and the extrainfo + document's signature will be verified. :rtype: dict :returns: A dictionary mapping bridge fingerprints to deduplicated - :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`s. + :api:`stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor`s. """ descriptors = [] - descriptorType = 'bridge-extra-info 1.1' - validate = False - if ('validate' in kwargs) and (kwargs['validate'] is True): - validate = True + # The ``stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor`` + # class (with ``descriptorType = 'bridge-extra-info 1.1``) is unsuitable + # for our purposes for the following reasons: + # + # 1. It expects a ``router-digest`` line, which is only present in + # sanitised bridge extrainfo descriptors. + # + # 2. It doesn't check the ``router-signature`` (nor does it expect there + # to be a signature). + descriptorType = 'extra-info 1.0' + + validate = True + if ('validate' in kwargs) and (kwargs['validate'] is False): + validate = False for filename in filenames: logging.info("Parsing %s descriptors with Stem: %s" diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 953f371..c3772c7 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -21,7 +21,7 @@ HAS_STEM = False try: from stem.descriptor.server_descriptor import RelayDescriptor - from stem.descriptor.extrainfo_descriptor import BridgeExtraInfoDescriptor + from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb.parse import descriptors except (ImportError, NameError), error: @@ -252,7 +252,7 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] - self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) + self.assertIsInstance(bridge, RelayExtraInfoDescriptor) def test_parse_descriptors_parseBridgeExtraInfoFiles_one_file(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with only one

1 0

[bridgedb/develop] Rewrite b.p.d.parseNetworkStatusFile() to use RouterStatusEntryV3.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit aa2d411c046318d541ef62d68c7d3da70918902b Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Aug 29 05:31:24 2014 +0000 Rewrite b.p.d.parseNetworkStatusFile() to use RouterStatusEntryV3. See Damian's comments on how we were using a RouterStatusEntryV2 in https://trac.torproject.org/projects/tor/ticket/9380#comment:16. --- lib/bridgedb/parse/descriptors.py | 73 ++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index a0806e2..5028ad3 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -16,48 +16,55 @@ import datetime import logging from stem.descriptor import extrainfo_descriptor -from stem.descriptor import networkstatus from stem.descriptor import server_descriptor from stem.descriptor import parse_file +from stem.descriptor.router_status_entry import _parse_file as _parseNSFile +from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb import safelog -def parseNetworkStatusFile(filename, validate=True): +def parseNetworkStatusFile(filename, validate=True, skipHeaders=True, + descriptorClass=RouterStatusEntryV3): """Parse a file which contains an ``@type bridge-networkstatus`` document. - :rtype: dict - :returns: A dictionary fingerprints mapped to - :api:`stem.descriptor.router_status_entry.RouterStatusEntryV2`s. + See `ticket #12254 <https://bugs.torproject.org/12254>`__ for why + networkstatus-bridges documents don't look anything like the networkstatus + v2 documents that they are purported to look like. They are missing all + headers, and the entire footer including authority signatures. + + :param str filename: The location of the file containing bridge + networkstatus descriptors. + :param bool validate: Passed along to Stem's parsers. If ``True``, the + descriptors will raise exceptions if they do not meet some definition + of correctness. + :param bool skipHeaders: If ``True``, skip parsing everything before the + first ``r`` line. + :param descriptorClass: A class (probably from + :api:`stem.descriptors.router_status_entry`) which Stem will parse + each descriptor it reads from **filename** into. + :raises ValueError: if the contents of a descriptor are malformed and + **validate** is ``True``. + :raises IOError: if the file at **filename** can't be read. + :rtype: list + :returns: A list of + :api:`stem.descriptor.router_status_entry.RouterStatusEntryV#`s. """ - logging.info("Parsing networkstatus entries with Stem: %s" % filename) - - fh = open(filename) - descriptors = fh.read() - fh.close() - - # See ticket #12254 for why networkstatus-bridges documents don't look - # anything like the networkstatus v2 documents that they are purported to - # look like. They are missing all headers, and the entire footer including - # authority signatures. - # - # https://trac.torproject.org/projects/tor/ticket/12254 - # - # As such, they do not currently start with a "published" line with an - # ISO8601 timestamp, as stem expects them to: - # - if not descriptors.startswith("published"): - precise = datetime.datetime.now().isoformat(sep=chr(0x20)) - timestamp = precise.rsplit('.', 1)[0] - descriptors = "published {t}\n{d}".format(t=timestamp, d=descriptors) - else: - logging.warn( - ("Networkstatus file '%s' started with 'published' line! Please " - "revise this function!") % filename) - - document = networkstatus.BridgeNetworkStatusDocument(descriptors, - validate=validate) - return document.routers + routers = [] + + logging.info("Parsing networkstatus file: %s" % filename) + with open(filename) as fh: + position = fh.tell() + if skipHeaders: + while not fh.readline().startswith('r '): + position = fh.tell() + logging.debug("Skipping %d bytes of networkstatus file." % position) + document = _parseNSFile(fh, validate, entry_class=descriptorClass, + start_position=position) + routers.extend(list(document)) + logging.info("Closed networkstatus file: %s" % filename) + + return routers def parseServerDescriptorsFile(filename, validate=False): """Parse a file which contains ``@type bridge-server-descriptor``s.

1 0

[bridgedb/develop] Rewrite Bridge class.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 170d391e5b456fc1f66b2d01d2aaa2892ea85a78 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 4 22:12:24 2014 +0000 Rewrite Bridge class. The bridge class is now capable of storing additional information which we might find useful about a bridge. Additionally, there are now methods for updating a Bridge instance's attribute via descriptors parsed with Stem. --- lib/bridgedb/bridges.py | 301 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 65d4d02..1ab88f1 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -31,6 +31,17 @@ class InvalidPluggableTransportIP(MalformedBridgeInfo): """Raised when a :class:`PluggableTransport` has an invalid address.""" +class ServerDescriptorDigestMismatch(MalformedBridgeInfo): + """Raised when the digest in an ``@type bridge-networkstatus`` document + doesn't match the hash digest of the ``@type bridge-server-descriptor``'s + contents. + """ + +class ServerDescriptorWithoutNetworkstatus(MalformedBridgeInfo): + """Raised when we find a ``@type bridge-server-descriptor`` which was not + mentioned in the latest ``@type bridge-networkstatus`` document. + """ + class PluggableTransport(object): """A single instance of a Pluggable Transport (PT) offered by a :class:`Bridge`. @@ -288,3 +299,293 @@ class PluggableTransport(object): self.port = kitchenSink[1] self.arguments = self._parseArgumentsIntoDict(kitchenSink[2]) self._runChecks() + + +class Bridge(object): + """A single bridge, and all the information we have for it. + + :type fingerprint: str or None + :ivar fingerprint: + + :type nickname: str or None + :ivar nickname: + + :ivar orPort: int or None + :ivar orPort: + + :ivar socksPort: int + :ivar socksPort: + + :type dirPort: int + :ivar dirPort: + + :type orAddresses: list + :ivar orAddresses: + + :type transports: list + :ivar transports: + + :type flags: :class:`~bridgedb.bridges.Flags` + :ivar flags: + + :type hibernating: bool + :ivar hibernating: + + :type contact: str or None + :ivar contact: The contact information for the this bridge's operator. + + :type platform: str or None + :ivar platform: The ``platform`` line taken from the + ``@type bridge-server-descriptor``, e.g. + ``'Tor 0.2.5.4-alpha on Linux'``. + + :type family: set or None + :ivar family: The fingerprints of other bridges related to this one. + """ + #: (bool) If ``True``, check that the signature of the bridge's + #: ``@type bridge-server-descriptor`` is valid and that the signature was + #: created with the ``signing-key`` contained in that descriptor. + _checkServerDescriptorSignature = True + + def __init__(self): + """Create a new ``Bridge``.""" + self.fingerprint = None + self.nickname = None + self.address = None + self.orPort = None + self.socksPort = 0 # Bridges should always have ``SOCKSPort`` and + self.dirPort = 0 # ``DirPort`` set to ``0`` + self.orAddresses = [] + self.transports = [] + self.flags = BridgeFlags() + self.hibernating = False + + self.bandwidth = None + self.bandwidthAverage = None + self.bandwidthBurst = None + self.bandwidthObserverd = None + + self.contact = None + self.family = None + self.platform = None + self.software = None + self.os = None + self.uptime = None + + self.onionKey = None + self.ntorOnionKey = None + self.signingKey = None + + self.descriptors = {'networkstatus': None, + 'server': None, + 'extrainfo': None} + + #: The hash digest of this bridge's ``@type bridge-server-descriptor``, + #: as signed (but not including the signature). This is found in the + #: 'r'-line of this bridge's ``@type bride-networkstatus`` document, + #: however it is stored here re-encoded from base64 into hexadecimal, + #: and converted to uppercase. + self.descriptorDigest = None + self.extrainfoDigest = None + + def _checkServerDescriptor(self, descriptor): + # If we're parsing the server-descriptor, require a networkstatus + # document: + if not self.descriptors['networkstatus']: + raise ServerDescriptorWithoutNetworkstatus( + ("We received a server-descriptor for bridge '%s' which has " + "no corresponding networkstatus document.") % + descriptor.fingerprint) + + ns = self.descriptors['networkstatus'] + + # We must have the digest of the server-descriptor from the + # networkstatus document: + if not self.descriptorDigest: + raise MissingServerDescriptorDigest( + ("The server-descriptor digest was missing from networkstatus " + "document for bridge '%s'.") % descriptor.fingerprint) + + digested = descriptor.digest() + # The digested server-descriptor must match the digest reported by the + # BridgeAuthority in the bridge's networkstatus document: + if not self.descriptorDigest == digested: + raise ServerDescriptorDigestMismatch( + ("The server-descriptor digest for bridge '%s' doesn't match " + "the digest reported by the BridgeAuthority in the " + "networkstatus document: \n" + "Digest reported in networkstatus: %s\n" + "Actual descriptor digest: %s\n") % + (descriptor.fingerprint, self.descriptorDigest, digested)) + + def _updateORAddresses(self, orAddresses): + for (address, port, ipVersion) in orAddresses: + if not ipVersion: # `False` means IPv4; `True` means IPv6. + # See https://bugs.torproject.org/9380#comment:27 + warnings.warn(FutureWarning( + ("Got IPv4 address in 'a'/'or-address' line! " + "Desriptor format may have changed!"))) + self.orAddresses.append(tuple([address, port])) + + def assertOK(self): + """Perform some additional validation on this bridge's info. + + We require that: + + 1. This bridge's :data:`fingerprint` is valid, accoring to + :func:`~bridgedb.parse.fingerprint.isValidFingerprint`. + + 2. This bridge's :data:`address` and any IP addresses contained in + :data:`orAddresses` are valid, according to + :func:`~bridgedb.parse.addr.isValidIP`. + + 3. The :data:`orPort` and any ports in :data:`orAddresses` are + between ``1`` and ``65535`` (inclusive). + + :raises MalformedBridgeInfo: if something was found to be malformed or + invalid. + """ + malformed = [] + + if not isValidFingerprint(self.fingerprint): + malformed.append("Invalid fingerprint: '%s'" % self.fingerprint) + if not isValidIP(self.address): + malformed.append("Invalid ORPort address: '%s'" % self.address) + if not (1 <= self.orPort <= 65535): + malformed.append("Invalid ORPort port: '%d'" % self.orPort) + for (address, port) in self.orAddresses: + if not isValidIP(address): + malformed.append("Invalid ORAddress address: '%s'" % address) + if not (1 <= port <= 65535): + malformed.append("Invalid ORAddress port: '%d'" % port) + + if malformed: + raise MalformedBridgeInfo('\n'.join(malformed)) + + def getDescriptorLastPublished(self): + """Get the timestamp for when this bridge's last known server + descriptor was published. + + :rtype: :type:`datetime.datetime` or ``None`` + :returns: A datetime object representing the timestamp of when the + last known ``@type bridge-server-descriptor`` was published, or + ``None`` if we have never seen a server descriptor for this + bridge. + """ + return getattr(self.descriptors['server'], 'published', None) + + def getExtrainfoLastPublished(self): + """Get the timestamp for when this bridge's last known extrainfo + descriptor was published. + + :rtype: :type:`datetime.datetime` or ``None`` + :returns: A datetime object representing the timestamp of when the + last known ``@type bridge-extrainfo`` descriptor was published, or + ``None`` if we have never seen an extrainfo descriptor for this + bridge. + """ + return getattr(self.descriptors['extrainfo'], 'published', None) + + def getNetworkstatusLastPublished(self): + """Get the timestamp for when this bridge's last known networkstatus + descriptor was published. + + :rtype: :type:`datetime.datetime` or ``None`` + :returns: A datetime object representing the timestamp of when the + last known ``@type networkstatus-bridge`` document was published, + or ``None`` if we have never seen a networkstatus document for + this bridge. + """ + return getattr(self.descriptors['networkstatus'], 'published', None) + + def updateFromNetworkstatus(self, descriptor): + """Update this bridge's attributes from a parsed networkstatus + descriptor. + + :type ns: :api:`stem.descriptors.router_status_entry.RouterStatusEntry` + :param ns: + """ + self.descriptors['networkstatus'] = descriptor + + # These fields are *only* found in the networkstatus document: + self.descriptorDigest = descriptor.digest + self.flags.update(descriptor.flags) + self.bandwidth = descriptor.bandwith + + # These fields are also found in the server-descriptor. We will prefer + # to use the information taken later from the server-descriptor + # because it is signed by the bridge. However, for now, we harvest all + # the info we can: + self.fingerprint = descriptor.fingerprint + self.nickname = descriptor.nickname + self.address = descriptor.address + self.orPort = descriptor.or_port + + self._updateORAddresses(descriptor.or_addresses) + + def updateFromServerDescriptor(self, descriptor): + """Update this bridge's info from an ``@type bridge-server-descriptor``. + + .. info:: + If :func:`~bridgedb.parse.descriptor.parseServerDescriptorFile` is + called with ``validate=True``, then Stem will handle checking that + the ``signing-key`` hashes to the ``fingerprint``. Stem will also + check that the ``router-signature`` on the descriptor is valid, + was created with the ``signing-key``, and is a signature of the + correct digest of the descriptor document (it recalculates the + digest for the descriptor to ensure that the signed one and the + actual digest match). + + :type descriptor: + :api:`stem.descriptor.server_descriptor.RelayDescriptor` + :param descriptor: + """ + self.descriptors['server'] = descriptor + + try: + self._checkServerDescriptor(descriptor) + except ValueError as error: + logging.warn(error) + # XXX should we throw away this descriptor? + + # Replace the values which we harvested from the networkstatus + # descriptor, because that one isn't signed with the bridge's identity + # key. + self.fingerprint = descriptor.fingerprint + self.address = descriptor.address + self.nickname = descriptor.nickname + self.orPort = descriptor.or_port + self._updateORAddresses(descriptor.or_addresses) + self.hibernating = descriptor.hibernating + + self.onionKey = descriptor.onion_key + self.ntorOnionKey = descriptor.ntor_onion_key + self.signingKey = descriptor.signing_key + + self.bandwidthAverage = descriptor.average_bandwidth + self.bandwidthBurst = descriptor.burst_bandwidth + self.bandwidthObserved = descriptor.bandwidth_observed + + self.contact = descriptor.contact + self.family = descriptor.family + self.platform = descriptor.platform + self.software = descriptor.tor_version + self.os = descriptor.operating_system + self.uptime = descriptor.uptime + + self.extrainfoDigest = descriptor.extrainfoDigest + + def updateFromExtraInfoDescriptor(self, descriptor): + """Update this bridge's information from an extrainfo descriptor. + + .. todo:: The ``transport`` attribute of Stem's + ``BridgeExtraInfoDescriptor`` class is a dictionary that uses the + Pluggable Transport's eype as the keys. Meaning that if a bridge + were to offer four instances of ``obfs3``, only one of them would + get to us through Stem. This might pose a problem someday. + + :type descriptor: + :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor` + :param descriptor: DOCDOC + """ + self.descriptors['extrainfo'] = descriptor

1 0

[bridgedb/develop] Deprecate b.Bridges.PluggableTransport for b.bridges.PluggableTransport.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit eeb5b7c9d72a4758f8c5dc70b67c8233548c91bf Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Sep 7 00:04:17 2014 +0000 Deprecate b.Bridges.PluggableTransport for b.bridges.PluggableTransport. This replaces the legacy `bridgedb.Bridges.PluggableTransport` class, which had several issues, including: - `assert` statements without catching errors - Cyclically referential data structures which contain __del__() methods. (Which means that these structures cannot be garbage collected!! see https://stackoverflow.com/a/10962484) * ADD new `bridgedb.bridges` module for replacing old code in `bridgedb.Bridges`. * MOVE legacy implementation `bridgedb.Bridges.PluggableTransport` to `bridgedb.test.deprecated` module for automated regression testing. * ADD new implementation, `bridgedb.bridges.PluggableTransport`, which is mostly backward-compatible with the old implementation, except that instead of taking an entire `bridgedb.Bridges.Bridge` class as its first argument, it takes just the uppercased, hexadecimal fingerprint of that bridge. This was done because the legacy `Bridge` and `PluggableTransport` classes posed a serialization problem due to infinitely self-referencing eachother, i.e. the `bridgedb.Bridges.Bridge.transports` attribute was a list of `bridgedb.Bridges.PluggableTransport`s, which in turn referenced their `bridgedb.Bridges.Bridge`, which referenced a list of bridgedb.Bridges.PluggableTransport`s... turtles all the way down, and they never get garbage collected, they just end up in Python's `gc.garbage` list for "the programmer to manually deal with". * FIXES part of #12505 https://bugs.torproject.org/12505 --- lib/bridgedb/Bridges.py | 104 ------------- lib/bridgedb/Main.py | 23 ++- lib/bridgedb/bridges.py | 290 +++++++++++++++++++++++++++++++++++++ lib/bridgedb/test/deprecated.py | 112 ++++++++++++++ lib/bridgedb/test/test_Bridges.py | 7 +- lib/bridgedb/test/test_Tests.py | 2 + 6 files changed, 424 insertions(+), 114 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index fa718c0..dead9f2 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -451,110 +451,6 @@ re_ipv6 = re.compile("\[([a-fA-F0-9:]+)\]:(.*$)") re_ipv4 = re.compile("((?:\d{1,3}\.?){4}):(.*$)") -class PluggableTransport(object): - """A PT with reference to the parent bridge on which it is running.""" - - def __init__(self, bridge, methodname, address, port, argdict=None): - """Create a ``PluggableTransport`` describing a PT running on a bridge. - - Pluggable transports are described within a bridge's ``@type - bridge-extrainfo`` descriptor, see the ``Specifications: Client - behavior`` section and the ``TOR_PT_SERVER_TRANSPORT_OPTIONS`` - description in pt-spec.txt_ for additional specification. - - :type bridge: :class:`Bridge` - :param bridge: The parent bridge running this pluggable transport - instance, i.e. the main ORPort bridge whose - ``@type bridge-server-descriptor`` contains a hash digest for a - ``@type bridge-extrainfo-document``, the latter of which contains - the parameter of this pluggable transport in its ``transport`` - line. - - :param str methodname: The canonical "name" for this pluggable - transport, i.e. the one which would be specified in a torrc - file. For example, ``"obfs2"``, ``"obfs3"``, ``"scramblesuit"`` - would all be pluggable transport method names. - - :param str address: The IP address of the transport. Currently (as of - 20 March 2014), there are no known, widely-deployed pluggable - transports which support IPv6. Ergo, this is very likely going to - be an IPv4 address. - - :param int port: A integer specifying the port which this pluggable - transport is listening on. (This should likely be whatever port the - bridge specified in its ``ServerTransportPlugin`` torrc line, - unless the pluggable transport is running in "managed" mode.) - - :param dict argdict: Some PTs can take additional arguments, which - must be distributed to the client out-of-band. These are present - in the ``@type bridge-extrainfo-document``, in the ``transport`` - line like so:: - - METHOD SP ADDR ":" PORT SP [K=V[,K=V[,K=V[…]]]] - - where K is the **argdict** key, and V is the value. For example, - in the case of ``scramblesuit``, for which the client must supply - a shared secret to the ``scramblesuit`` instance running on the - bridge, the **argdict** would be something like:: - - {'password': 'NEQGQYLUMUQGK5TFOJ4XI2DJNZTS4LRO'} - - .. _pt-spec.txt: - https://gitweb.torproject.org/torspec.git/blob/HEAD:/pt-spec.txt - """ - #XXX: assert are disabled with python -O - assert isinstance(bridge, Bridge) - assert type(address) in (ipaddr.IPv4Address, ipaddr.IPv6Address) - assert type(port) is int - assert (0 < port < 65536) - assert type(methodname) is str - - self.bridge = bridge - self.address = address - self.port = port - self.methodname = methodname - if type(argdict) is dict: - self.argdict = argdict - else: self.argdict = {} - - def getTransportLine(self, includeFingerprint=False, bridgePrefix=False): - """Get a torrc line for this pluggable transport. - - This method does not return lines which are prefixed with the word - 'bridge', as they would be in a torrc file. Instead, lines returned - look like this: - - obfs3 245.102.100.252:23619 59ca743e89b508e16b8c7c6d2290efdfd14eea98 - - :param bool includeFingerprints: If ``True``, include the digest of - this bridges public identity key in the torrc line. - :param bool bridgePrefix: If ``True``, add ``'Bridge '`` to the - beginning of each returned line (suitable for pasting directly - into a torrc file). - :rtype: str - :returns: A configuration line for adding this pluggable transport - into a torrc file. - """ - sections = [] - - if bridgePrefix: - sections.append('Bridge') - - if isinstance(self.address, ipaddr.IPv6Address): - host = "%s [%s]:%d" % (self.methodname, self.address, self.port) - else: - host = "%s %s:%d" % (self.methodname, self.address, self.port) - sections.append(host) - - if includeFingerprint: - sections.append(self.bridge.fingerprint) - - args = " ".join(["%s=%s" % (k, v) for k, v in self.argdict.items()]) - sections.append(args) - - line = ' '.join(sections) - return line - def parseExtraInfoFile(f): """ parses lines in Bridges extra-info documents. diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index ce3edc5..e1963f5 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -27,6 +27,9 @@ from bridgedb import proxy from bridgedb import safelog from bridgedb import schedule from bridgedb import util +from bridgedb.bridges import InvalidPluggableTransportIP +from bridgedb.bridges import MalformedPluggableTransport +from bridgedb.bridges import PluggableTransport from bridgedb.configure import loadConfig from bridgedb.configure import Conf from bridgedb.parse import options @@ -128,13 +131,19 @@ def load(state, splitter, clear=False): if bridges[ID].running: logging.info("Adding %s transport to running bridge" % method_name) - bridgePT = Bridges.PluggableTransport( - bridges[ID], method_name, address, port, argdict) - bridges[ID].transports.append(bridgePT) - if not bridgePT in bridges[ID].transports: - logging.critical( - "Added a transport, but it disappeared!", - "\tTransport: %r" % bridgePT) + try: + bridgePT = PluggableTransport(Bridges.toHex(ID), + method_name, address, + port, argdict) + except (InvalidPluggableTransportIP, + MalformedPluggableTransport) as error: + logging.warn(error) + else: + bridges[ID].transports.append(bridgePT) + if not bridgePT in bridges[ID].transports: + logging.critical( + "Added a transport, but it disappeared!", + "\tTransport: %r" % bridgePT) except KeyError as error: logging.error("Could not find bridge with fingerprint '%s'." % Bridges.toHex(ID)) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py new file mode 100644 index 0000000..65d4d02 --- /dev/null +++ b/lib/bridgedb/bridges.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_bridges -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: please see the AUTHORS file for attributions +# :copyright: (c) 2013-2014, Isis Lovecruft +# (c) 2007-2014, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +"""Classes for manipulating and storing Bridges and their attributes.""" + + +import ipaddr +import logging +import os + +from bridgedb.parse.addr import isValidIP +from bridgedb.parse.fingerprint import isValidFingerprint + + +class MalformedBridgeInfo(ValueError): + """Raised when some information about a bridge appears malformed.""" + + +class MalformedPluggableTransport(MalformedBridgeInfo): + """Raised when information used to initialise a :class:`PluggableTransport` + appears malformed. + """ + +class InvalidPluggableTransportIP(MalformedBridgeInfo): + """Raised when a :class:`PluggableTransport` has an invalid address.""" + + +class PluggableTransport(object): + """A single instance of a Pluggable Transport (PT) offered by a + :class:`Bridge`. + + Pluggable transports are described within a bridge's + ``@type bridge-extrainfo`` descriptor, see the + ``Specifications: Client behavior`` section and the + ``TOR_PT_SERVER_TRANSPORT_OPTIONS`` description in pt-spec.txt_ for + additional specification. + + .. _pt-spec.txt: + https://gitweb.torproject.org/torspec.git/blob/HEAD:/pt-spec.txt + + :type fingerprint: str + :ivar fingerprint: The uppercased, hexadecimal fingerprint of the identity + key of the parent bridge running this pluggable transport instance, + i.e. the main ORPort bridge whose ``@type bridge-server-descriptor`` + contains a hash digest for a ``@type bridge-extrainfo-document``, the + latter of which contains the parameter of this pluggable transport in + its ``transport`` line. + + :type methodname: str + :ivar methodname: The canonical "name" for this pluggable transport, + i.e. the one which would be specified in a torrc file. For example, + ``"obfs2"``, ``"obfs3"``, ``"scramblesuit"`` would all be pluggable + transport method names. + + :type address: ``ipaddr.IPv4Address`` or ``ipaddr.IPv6Address`` + :ivar address: The IP address of the transport. Currently (as of 20 March + 2014), there are no known, widely-deployed pluggable transports which + support IPv6. Ergo, this is very likely going to be an IPv4 address. + + :type port: int + :ivar port: A integer specifying the port which this pluggable transport + is listening on. (This should likely be whatever port the bridge + specified in its ``ServerTransportPlugin`` torrc line, unless the + pluggable transport is running in "managed" mode.) + + :type arguments: dict + :ivar arguments: Some PTs can take additional arguments, which must be + distributed to the client out-of-band. These are present in the + ``@type bridge-extrainfo-document``, in the ``transport`` line like + so:: + + METHOD SP ADDR ":" PORT SP [K=V[,K=V[,K=V[…]]]] + + where K is the key in **arguments**, and V is the value. For example, + in the case of ``scramblesuit``, for which the client must supply a + shared secret to the ``scramblesuit`` instance running on the bridge, + the **arguments** would be something like:: + + {'password': 'NEQGQYLUMUQGK5TFOJ4XI2DJNZTS4LRO'} + """ + + def __init__(self, fingerprint=None, methodname=None, + address=None, port=None, arguments=None): + """Create a ``PluggableTransport`` describing a PT running on a bridge. + + :param str fingerprint: The uppercased, hexadecimal fingerprint of the + identity key of the parent bridge running this pluggable transport. + :param str methodname: The canonical "name" for this pluggable + transport. See :data:`methodname`. + :param str address: The IP address of the transport. See + :data:`address`. + :param int port: A integer specifying the port which this pluggable + transport is listening on. + :param dict arguments: Any additional arguments which the PT takes, + which must be distributed to the client out-of-band. See + :data:`arguments`. + """ + self.fingerprint = fingerprint + self.address = address + self.port = port + self.methodname = methodname + self.arguments = arguments + + # Because we can intitialise this class with the __init__() + # parameters, or use the ``updateFromStemTransport()`` method, we'll + # only use the ``_runChecks()`` method now if we were initialised with + # parameters: + if (self.fingerprint or self.address or self.port or + self.methodname or self.arguments): + self._runChecks() + + def _parseArgumentsIntoDict(self, argumentList): + """Convert a list of Pluggable Transport arguments into a dictionary + suitable for :data:`arguments`. + + :param list argumentList: A list of Pluggable Transport + arguments. There might be multiple, comma-separated ``K=V`` + Pluggable Transport arguments in a single item in the + **argumentList**, or each item might be its own ``K=V``; we don't + care and we should be able to parse it either way. + :rtype: dict + :returns: A dictionary of all the ``K=V`` Pluggable Transport + arguments. + """ + argDict = {} + + # PT argumentss are comma-separated in the extrainfo + # descriptors. While there *shouldn't* be anything after them that was + # separated by a space (and hence would wind up being in a different + # item in `arguments`), if there was we'll join it to the rest of the + # PT arguments with a comma so that they are parsed as if they were PT + # arguments as well: + allArguments = ','.join(argumentList) + + for arg in allArguments.split(','): + try: + key, value = arg.split('=') + except ValueError: + logging.warn(" Couldn't parse K=V from PT arg: %r" % arg) + else: + logging.debug(" Parsed PT Argument: %s: %s" % (key, value)) + argDict[key] = value + + return argDict + + def _runChecks(self): + """Validate that we were initialised with acceptable parameters. + + We currently check that: + + 1. The :data:`fingerprint` is valid, according to + :func:`~bridgedb.parse.fingerprint.isValidFingerprint`. + + 2. The :data:`address` is valid, according to + :func:`~bridgedb.parse.addr.isValidIP`. + + 3. The :data:`port` is an integer, and that it is between the values + of ``0`` and ``65535`` (inclusive). + + 4. The :data:`arguments` is a dictionary. + + :raises MalformedPluggableTransport: if any of the above checks fails. + """ + if not isValidFingerprint(self.fingerprint): + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with bad Bridge " + "fingerprint: %r.") % self.fingerprint) + + valid = isValidIP(self.address) + if not valid: + raise InvalidPluggableTransportIP( + ("Cannot create PluggableTransport with address '%s'. " + "type(address)=%s.") % (self.address, type(self.address))) + self.address = ipaddr.IPAddress(self.address) + + try: + # Coerce the port to be an integer: + self.port = int(self.port) + except TypeError: + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with port type: %s.") + % type(self.port)) + else: + if not (0 <= self.port <= 65535): + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with out-of-range port:" + " %r.") % self.port) + + if not isinstance(self.arguments, dict): + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with arguments type: %s") + % type(self.arguments)) + + def getTransportLine(self, includeFingerprint=True, bridgePrefix=False): + """Get a Bridge Line for this :class:`PluggableTransport`. + + .. glossary:: + + Bridge Line + A "Bridge Line" is how BridgeDB refers to lines in a ``torrc`` + file which should begin with the word ``"Bridge"``, and it is how + a client tells their Tor process that they would like to use a + particular bridge. + + .. note:: If **bridgePrefix** is ``False``, this method does not + return lines which are prefixed with the word 'bridge', as they + would be in a torrc file. Instead, lines returned look like this:: + + obfs3 245.102.100.252:23619 59ca743e89b508e16b8c7c6d2290efdfd14eea98 + + This was made configurable to fix Vidalia being a brain-damaged + piece of shit (#5851_). TorLaucher replaced Vidalia soon after, + and TorLauncher is intelligent enough to understand + :term:`Bridge Line`s regardless of whether or not they are prefixed + with the word "Bridge". + + .. _#5851: https://bugs.torproject.org/5851 + + :param bool includeFingerprints: If ``True``, include the digest of + this bridges public identity key in the torrc line. + :param bool bridgePrefix: If ``True``, add ``'Bridge '`` to the + beginning of each returned line (suitable for pasting directly + into a ``torrc`` file). + :rtype: str + :returns: A configuration line for adding this Pluggable Transport + into a ``torrc`` file. + """ + sections = [] + + if bridgePrefix: + sections.append('Bridge') + + if self.address.version == 6: + # If the address was IPv6, put brackets around it: + host = '%s [%s]:%d' % (self.methodname, self.address, self.port) + else: + host = '%s %s:%d' % (self.methodname, self.address, self.port) + sections.append(host) + + if includeFingerprint: + sections.append(self.fingerprint) + + for key, value in self.arguments.items(): + sections.append('%s=%s' % (key, value)) + + line = ' '.join(sections) + + return line + + def updateFromStemTransport(self, fingerprint, methodname, kitchenSink): + """Update this :class:`PluggableTransport` from the data structure + which Stem uses. + + Stem's + :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor` + parses extrainfo ``transport`` lines into a dictionary with the + following structure:: + + {u'obfs2': (u'34.230.223.87', 37339, []), + u'obfs3': (u'34.230.223.87', 37338, []), + u'obfs4': (u'34.230.223.87', 37341, [ + (u'iat-mode=0,' + u'node-id=2a79f14120945873482b7823caabe2fcde848722,' + u'public-key=0a5b046d07f6f971b7776de682f57c5b9cdc8fa060db7ef59de82e721c8098f4')]), + u'scramblesuit': (u'34.230.223.87', 37340, [ + u'password=ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'])} + + This method will initialise this class from the dictionary key + (**methodname**) and its tuple of values (**kitchenSink**). + + :param str fingerprint: The uppercased, hexadecimal fingerprint of the + identity key of the parent bridge running this pluggable transport. + :param str methodname: The :data:`methodname` of this Pluggable + Transport. + :param tuple kitchenSink: Everything else that was on the + ``transport`` line in the bridge's extrainfo descriptor, which + Stem puts into the 3-tuples shown in the example above. + """ + self.fingerprint = fingerprint + self.methodname = methodname + self.address = kitchenSink[0] + self.port = kitchenSink[1] + self.arguments = self._parseArgumentsIntoDict(kitchenSink[2]) + self._runChecks() diff --git a/lib/bridgedb/test/deprecated.py b/lib/bridgedb/test/deprecated.py index a972a52..40cfe0a 100644 --- a/lib/bridgedb/test/deprecated.py +++ b/lib/bridgedb/test/deprecated.py @@ -123,6 +123,118 @@ class ParseORAddressError(Exception): @deprecate.deprecated( + Version('bridgedb', 0, 2, 4), + replacement='bridgedb.bridges.PluggableTransport') +class PluggableTransport(object): + """A PT with reference to the parent bridge on which it is running. + + Deprecated :class:`bridgedb.Bridges.PluggableTransport`, replaced in + bridgedb-0.2.4, by :class:`bridgedb.bridges.PluggableTransport`. + """ + + def __init__(self, bridge, methodname, address, port, argdict=None): + """Create a ``PluggableTransport`` describing a PT running on a bridge. + + Pluggable transports are described within a bridge's ``@type + bridge-extrainfo`` descriptor, see the ``Specifications: Client + behavior`` section and the ``TOR_PT_SERVER_TRANSPORT_OPTIONS`` + description in pt-spec.txt_ for additional specification. + + :type bridge: :class:`Bridge` + :param bridge: The parent bridge running this pluggable transport + instance, i.e. the main ORPort bridge whose + ``@type bridge-server-descriptor`` contains a hash digest for a + ``@type bridge-extrainfo-document``, the latter of which contains + the parameter of this pluggable transport in its ``transport`` + line. + + :param str methodname: The canonical "name" for this pluggable + transport, i.e. the one which would be specified in a torrc + file. For example, ``"obfs2"``, ``"obfs3"``, ``"scramblesuit"`` + would all be pluggable transport method names. + + :param str address: The IP address of the transport. Currently (as of + 20 March 2014), there are no known, widely-deployed pluggable + transports which support IPv6. Ergo, this is very likely going to + be an IPv4 address. + + :param int port: A integer specifying the port which this pluggable + transport is listening on. (This should likely be whatever port the + bridge specified in its ``ServerTransportPlugin`` torrc line, + unless the pluggable transport is running in "managed" mode.) + + :param dict argdict: Some PTs can take additional arguments, which + must be distributed to the client out-of-band. These are present + in the ``@type bridge-extrainfo-document``, in the ``transport`` + line like so:: + + METHOD SP ADDR ":" PORT SP [K=V[,K=V[,K=V[…]]]] + + where K is the **argdict** key, and V is the value. For example, + in the case of ``scramblesuit``, for which the client must supply + a shared secret to the ``scramblesuit`` instance running on the + bridge, the **argdict** would be something like:: + + {'password': 'NEQGQYLUMUQGK5TFOJ4XI2DJNZTS4LRO'} + + .. _pt-spec.txt: + https://gitweb.torproject.org/torspec.git/blob/HEAD:/pt-spec.txt + """ + #XXX: assert are disabled with python -O + assert isinstance(bridge, Bridge) + assert type(address) in (ipaddr.IPv4Address, ipaddr.IPv6Address) + assert type(port) is int + assert (0 < port < 65536) + assert type(methodname) is str + + self.bridge = bridge + self.address = address + self.port = port + self.methodname = methodname + if type(argdict) is dict: + self.argdict = argdict + else: self.argdict = {} + + def getTransportLine(self, includeFingerprint=False, bridgePrefix=False): + """Get a torrc line for this pluggable transport. + + This method does not return lines which are prefixed with the word + 'bridge', as they would be in a torrc file. Instead, lines returned + look like this: + + obfs3 245.102.100.252:23619 59ca743e89b508e16b8c7c6d2290efdfd14eea98 + + :param bool includeFingerprints: If ``True``, include the digest of + this bridges public identity key in the torrc line. + :param bool bridgePrefix: If ``True``, add ``'Bridge '`` to the + beginning of each returned line (suitable for pasting directly + into a torrc file). + :rtype: str + :returns: A configuration line for adding this pluggable transport + into a torrc file. + """ + sections = [] + + if bridgePrefix: + sections.append('Bridge') + + if isinstance(self.address, ipaddr.IPv6Address): + host = "%s [%s]:%d" % (self.methodname, self.address, self.port) + else: + host = "%s %s:%d" % (self.methodname, self.address, self.port) + sections.append(host) + + if includeFingerprint: + sections.append(self.bridge.fingerprint) + + args = " ".join(["%s=%s" % (k, v) for k, v in self.argdict.items()]) + sections.append(args) + + line = ' '.join(sections) + return line + + +(a)deprecate.deprecated( Version('bridgedb', 0, 0, 1), replacement='bridgedb.parse.addr.PortList') class PortList: diff --git a/lib/bridgedb/test/test_Bridges.py b/lib/bridgedb/test/test_Bridges.py index 93d6c10..2c7d09b 100644 --- a/lib/bridgedb/test/test_Bridges.py +++ b/lib/bridgedb/test/test_Bridges.py @@ -15,6 +15,7 @@ from binascii import a2b_hex from twisted.trial import unittest from bridgedb import Bridges +from bridgedb.bridges import PluggableTransport from bridgedb.parse.addr import PortList import hashlib @@ -125,9 +126,9 @@ class BridgeClassTest(unittest.TestCase): id_digest=self.id_digest, or_addresses=self.or_addresses) ptArgs = {'password': 'NEQGQYLUMUQGK5TFOJ4XI2DJNZTS4LRO'} - pt = Bridges.PluggableTransport(bridge, 'scramblesuit', - ipaddr.IPAddress('42.42.42.42'), 4242, - ptArgs) + pt = PluggableTransport(bridge.fingerprint, 'scramblesuit', + ipaddr.IPAddress('42.42.42.42'), 4242, + ptArgs) bridge.transports.append(pt) bridgeLine = bridge.getConfigLine(includeFingerprint=True, transport='scramblesuit') diff --git a/lib/bridgedb/test/test_Tests.py b/lib/bridgedb/test/test_Tests.py index 816cd40..e57f4a1 100644 --- a/lib/bridgedb/test/test_Tests.py +++ b/lib/bridgedb/test/test_Tests.py @@ -94,6 +94,8 @@ def monkeypatchTests(): patcher.addPatch(Tests.bridgedb.Bridges, 'fromHex', binascii.a2b_hex) patcher.addPatch(Tests.bridgedb.Bridges, 'is_valid_fingerprint', deprecated.is_valid_fingerprint) + patcher.addPatch(Tests.bridgedb.Bridges, 'PluggableTransport', + deprecated.PluggableTransport) return patcher

1 0

[bridgedb/develop] Replace descriptors in test_parse_descriptors with newer leekspin ones.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 74f8a5548fb408c64169321789f7bdb718b3996b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 00:55:55 2014 +0000 Replace descriptors in test_parse_descriptors with newer leekspin ones. The descriptors in this unittest file were created by an older version of Leekspin, which had a bugs in the way it created signatures and coverted between fingerprint formats. In addition, we're now able to test parsing of obfs4 `transport` lines. --- lib/bridgedb/parse/descriptors.py | 2 +- lib/bridgedb/test/test_parse_descriptors.py | 150 +++++++++++++++------------ 2 files changed, 82 insertions(+), 70 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 6c6fd3e..a7cb202 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -66,7 +66,7 @@ def parseNetworkStatusFile(filename, validate=True, skipAnnotations=True, return routers -def parseServerDescriptorsFile(filename, validate=False): +def parseServerDescriptorsFile(filename, validate=True): """Parse a file which contains ``@type bridge-server-descriptor``s. .. note:: ``validate`` defaults to ``False`` because there appears to be a diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 1abd08f..953f371 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -31,111 +31,114 @@ else: BRIDGE_NETWORKSTATUS_0 = '''\ -r OutwitsPlod b6khbPOgbomgMSGswx9w+N/X3cw qBckYSWbwl/F/qzQWwMJWBxHZ+w 2014-03-12 16:07:08 152.78.9.20 17810 0 -a [bfbd:7a90:2347:cc4:e854:64b3:2c31:124f]:17810 +r MiserLandfalls 4IsyTSCtChPhFPAnq5rD8yymlqA /GMC4lz8RXT/62v6kZNdmzSmopk 2014-11-04 06:23:22 2.215.61.223 4056 0 +a [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 s Fast Guard Running Stable Valid -w Bandwidth=1902273 +w Bandwidth=1678904 p reject 1-65535 ''' BRIDGE_NETWORKSTATUS_1 = '''\ -r Reestablishes jPqMRoqkH62eLMXl76DqIddlpto 2BETKn1sOghC6coUkCSq/9mvPNM 2014-08-20 19:52:41 25.178.4.186 32324 0 -a [d7b3:8c3e:186a:d65f:706:cbfd:8512:fd1]:32324 +r Unmentionable BgOrX0ViP5hNsK5ZvixAuPZ6EY0 NTg9NoE5ls9KjF96Dp/UdrabZ9Y 2014-11-04 12:23:37 80.44.173.87 51691 0 +a [da14:7d1e:ba8e:60d0:b078:3f88:382b:5c70]:51690 s Fast Guard Running Stable Valid -w Bandwidth=497963 +w Bandwidth=24361 p reject 1-65535 ''' BRIDGE_SERVER_DESCRIPTOR = '''\ -router OutwitsPlod 152.78.9.20 17810 0 0 -or-address [bfbd:7a90:2347:cc4:e854:64b3:2c31:124f]:17810 -platform Tor 0.2.4.16-rc on Linux -protocols Link 1 2 Circuit 1 -published 2014-03-12 16:07:08 -fingerprint 6FA9 216C F3A0 6E89 A031 21AC C31F 70F8 DFD7 DDCC -uptime 57032961 -bandwidth 2240117028 2532306205 1947927850 -extra-info-digest 069EBB610CD8B02BF1BB0CAB17B99DDA73CCC91A +@purpose bridge +router MiserLandfalls 2.215.61.223 4056 0 0 +or-address [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 +platform Tor 0.2.2.39 on Linux +opt protocols Link 1 2 Circuit 1 +published 2014-11-04 06:23:22 +opt fingerprint E08B 324D 20AD 0A13 E114 F027 AB9A C3F3 2CA6 96A0 +uptime 24247659 +bandwidth 1977077890 2234957615 1719198165 +opt extra-info-digest 1CBBB3D6158F324476E6804B7EE25623899271CB onion-key -----BEGIN RSA PUBLIC KEY----- -MIGkAgEAMA0GCSqGSIb3DQEBAQUABIGPMIGMAgEAAoGBANI67YIwW8xF2v310PZt -Qc8jm0ptwLHmgBdhAzHAIGagqknjvukX5GTL0zie5covhxrQhZqjJm/gQ8inwkol -kZCue1ZQ9PHaTWjz58ESMQo41h+9Whfd8Egm2ev1+MwqlPy1Kr3rcPNIEetsmtil -DFNocpEfq1MC0tDG6qVO6/FNAgMBAAE= +MIGJAoGBAOm4NX2wi8JmgcAyvOyiAEfq9UkzaNHK+VnSZBiPIrb5GAKFibR7S+Bb +7+x7tsT8VBNbe9QmwML2GVah3xXg68gJAksMNIgFdpud+zMhduuGd0jr7V55aLmH +ePGJYCh78B9RqfvmeTridp3pljwcAheKKH/YKi3nv1fPY0BwahurAgMBAAE= -----END RSA PUBLIC KEY----- signing-key -----BEGIN RSA PUBLIC KEY----- -MIGkAgEAMA0GCSqGSIb3DQEBAQUABIGPMIGMAgEAAoGBALk0Ws5qPlgwKO6IQ1b/ -aamtEdXEPj2DrZTF3aGYR5zoZgw9gwmkbRHjrMQ/Wj+QHg0cTFY2DsYt81QXwiv+ -m1P9sshMZSZZz2P8Ld8WqUNuN7YIIJx/fj9Vy6LRFySzoyQ4FF/1Dio+JD0rvtyc -ZyRJl2aV5iYA9/TQY2zs2cxFAgMBAAE= +MIGJAoGBANd/JkrTZRT24EkK3DDc/E+Nj1QBnKIm/xXMyW0gkotFOVdewIWjwQ5z +Tn3YbDhrFN0aFYVdVwNbRhW83e+jZDkpIQuxlQOx6bT13vrzmg8ff1tH8I9EePl7 +MO4v0DLPIEcu7Zfz90oC1bl36oqNsD4h0v4yK/XjVwLutIGiy3gTAgMBAAE= -----END RSA PUBLIC KEY----- -hidden-service-dir contact Somebody <somebody(a)example.com> +ntor-onion-key NBsk2O6ks5qnxLhhhKPd59zi0IzfjnakoOJP+Cm8OAE reject *:* router-signature -----BEGIN SIGNATURE----- -i/nkrD4VxqWcnAlBS48hIilrE7C4DvRJhN4XWep7TXNbEC48IqFG+49xpKV6qkts -yKaUDBfD9Y1tMM0mrRjEWK0xYWX/4Ug9Xbbv2q1so4EuS35AF11d69Yf/2ppnCu7 -r+qtX7csROF4KyFJYFNJUKf/hroPHKWuTGCcqzb+D68= +YYA5wJTHcjqXk/QBaDXHX/4Fb8W2OctF4X4VHyxH9Hsou4Ip7nzdfWzbBTcBiIrt +ybaaMO15L9Ctkli/capN+nCw2jWgivgiPnAmJNmLGeN6skTKjLPAau+839hBuQxu +P2aB/+XQfzFBA5TaWF83coDng4OGodhwHaOx10Kn7Bg= -----END SIGNATURE----- ''' BRIDGE_EXTRA_INFO_DESCRIPTOR = '''\ -extra-info OutwitsPlod 6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC -published 2014-03-12 16:07:08 -write-history 2014-03-12 16:07:08 (900 s) 3188736,2226176,2866176 -read-history 2014-03-12 16:07:08 (900 s) 3891200,2483200,2698240 -dirreq-write-history 2014-03-12 16:07:08 (900 s) 1024,0,2048 -dirreq-read-history 2014-03-12 16:07:08 (900 s) 0,0,0 -geoip-db-digest AAF7B842E52974556F9969A62DF8D31F9D886A33 -geoip6-db-digest C2C80F4EF2908E55A764603B08A8CB99A681EA19 -dirreq-stats-end 2014-03-12 16:07:08 (86400 s) +extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 +published 2014-11-04 06:23:22 +write-history 2014-11-04 06:23:22 (900 s) 3188736,2226176,2866176 +read-history 2014-11-04 06:23:22 (900 s) 3891200,2483200,2698240 +dirreq-write-history 2014-11-04 06:23:22 (900 s) 1024,0,2048 +dirreq-read-history 2014-11-04 06:23:22 (900 s) 0,0,0 +geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F +geoip6-db-digest E983833985E4BCA34CEF611B2DF51942D188E638 +dirreq-stats-end 2014-11-04 06:23:22 (86400 s) dirreq-v3-ips dirreq-v3-reqs dirreq-v3-resp ok=16,not-enough-sigs=0,unavailable=0,not-found=0,not-modified=0,busy=0 dirreq-v3-direct-dl complete=0,timeout=0,running=0 dirreq-v3-tunneled-dl complete=12,timeout=0,running=0 -transport obfs3 152.78.9.20:17811 -transport obfs2 152.78.9.20:17812 -bridge-stats-end 2014-03-12 16:07:08 (86400 s) +transport obfs3 2.215.61.223:4057 +transport obfs2 2.215.61.223:4058 +transport scramblesuit 2.215.61.223:4059 password=ABCDEFGHIJKLMNOPQRSTUVWXYZ234567 +transport obfs4 2.215.61.223:4060 iat-mode=0,node-id=19a448c01aa2e7d55979473b647e282459995b85,public-key=7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25 +bridge-stats-end 2014-11-04 06:23:22 (86400 s) bridge-ips ca=8 bridge-ip-versions v4=8,v6=0 bridge-ip-transports <OR>=8 router-signature -----BEGIN SIGNATURE----- -aW1ZlxqeaGcbNSxVpMONU0ER4xgdihb9X2crguzKa/TYVweCZ2Ew7x3Rsg4cUNpr -Fb05F3Zxg6ZUTMC8gfh6leDGw5eSX7OGVaaJICTfeLbNopLVk+JKNGMJ32R/Zia0 -feWndKJk/zj5ZtkMND8VVbWuJE+R6Jh2Q3L0p8IZ6J4= +KOXNPCoe+Q+thFA/Lz7RTja2tWp4oC6SvyIooEZibHtEDgiXuU4sELWT4bSOk3np +RVmu7QPMmNybx4LHowq3pOeNLtJzpWg8Pfo+N6tR+K4nqPwBRmpsuDhCD/tIXJlP +U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = '''\ -extra-info OutwitsPlod 6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC -published 2014-03-12 17:07:08 -write-history 2014-03-12 17:07:08 (900 s) 3188736,2226176,2866176 -read-history 2014-03-12 17:07:08 (900 s) 3891200,2483200,2698240 -dirreq-write-history 2014-03-12 17:07:08 (900 s) 1024,0,2048 -dirreq-read-history 2014-03-12 17:07:08 (900 s) 0,0,0 -geoip-db-digest AAF7B842E52974556F9969A62DF8D31F9D886A33 -geoip6-db-digest C2C80F4EF2908E55A764603B08A8CB99A681EA19 -dirreq-stats-end 2014-03-12 17:07:08 (86400 s) +extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 +published 2014-11-04 08:10:25 +write-history 2014-11-04 08:10:25 (900 s) 3188736,2226176,2866176,2226176 +read-history 2014-11-04 08:10:25 (900 s) 3891200,2483200,2698240,2483200 +dirreq-write-history 2014-11-04 08:10:25 (900 s) 1024,0,2048,3072 +dirreq-read-history 2014-11-04 08:10:25 (900 s) 0,0,0,0 +geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F +geoip6-db-digest E983833985E4BCA34CEF611B2DF51942D188E638 +dirreq-stats-end 2014-11-04 08:10:25 (86400 s) dirreq-v3-ips dirreq-v3-reqs dirreq-v3-resp ok=16,not-enough-sigs=0,unavailable=0,not-found=0,not-modified=0,busy=0 dirreq-v3-direct-dl complete=0,timeout=0,running=0 dirreq-v3-tunneled-dl complete=12,timeout=0,running=0 -transport obfs3 152.78.9.20:17811 -transport obfs2 152.78.9.20:17812 -bridge-stats-end 2014-03-12 17:07:08 (86400 s) +transport obfs3 2.215.61.223:4057 +transport obfs2 2.215.61.223:4058 +transport scramblesuit 2.215.61.223:4059 password=ABCDEFGHIJKLMNOPQRSTUVWXYZ234567 +transport obfs4 2.215.61.223:4060 iat-mode=0,node-id=19a448c01aa2e7d55979473b647e282459995b85,public-key=7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25 +bridge-stats-end 2014-11-04 08:10:25 (86400 s) bridge-ips ca=8 bridge-ip-versions v4=8,v6=0 bridge-ip-transports <OR>=8 router-signature -----BEGIN SIGNATURE----- -aW1ZlxqeaGcbNSxVpMONU0ER4xgdihb9X2crguzKa/TYVweCZ2Ew7x3Rsg4cUNpr -Fb05F3Zxg6ZUTMC8gfh6leDGw5eSX7OGVaaJICTfeLbNopLVk+JKNGMJ32R/Zia0 -feWndKJk/zj5ZtkMND8VVbWuJE+R6Jh2Q3L0p8IZ6J4= +KOXNPCoe+Q+thFA/Lz7RTja2tWp4oC6SvyIooEZibHtEDgiXuU4sELWT4bSOk3np +RVmu7QPMmNybx4LHowq3pOeNLtJzpWg8Pfo+N6tR+K4nqPwBRmpsuDhCD/tIXJlP +U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' @@ -147,6 +150,11 @@ class ParseDescriptorsTests(unittest.TestCase): def setUp(self): """Test if we have Stem installed. Skip these tests if it's missing.""" + self.expectedIPBridge0 = '2.215.61.223' + self.expectedIPBridge1 = '80.44.173.87' + + self.expectedFprBridge0 = 'E08B324D20AD0A13E114F027AB9AC3F32CA696A0' + if self.skip: raise unittest.SkipTest("Couldn't import Stem.") @@ -174,9 +182,8 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertIsInstance(routers, list) bridge = routers[0] self.assertIsInstance(bridge, RelayDescriptor) - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + self.assertEqual(bridge.address, self.expectedIPBridge0) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeNetworkStatusFile_return_type(self): """``b.p.descriptors.parseNetworkStatusFile`` should return a dict.""" @@ -210,14 +217,15 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + self.assertEqual(bridge.address, self.expectedIPBridge0) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeNetworkStatusFile_2(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with two bridge networkstatus descriptors. """ + expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] + # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -225,9 +233,9 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_1) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + + self.assertIn(bridge.address, expectedIPs) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeExtraInfoFiles_return_type(self): """The return type of ``b.p.descriptors.parseBridgeExtraInfoFiles`` @@ -253,9 +261,13 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] - self.assertEqual(len(bridge.transport), 2) - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + + # The number of transports we parsed should be equal to the number of + # 'transport' lines in the descriptor: + self.assertEqual(len(bridge.transport), + BRIDGE_EXTRA_INFO_DESCRIPTOR.count('transport ')) + + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeExtraInfoFiles_two_files(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with two

1 0

[bridgedb/develop] Add a utility function for saving copies of unparsable descriptor files.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 905eeee999d5be4f68c5fb323e809c756d1d828f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 02:30:55 2014 +0000 Add a utility function for saving copies of unparsable descriptor files. * ADD new function, ``b.p.descriptors._copyUnparseableDescriptorFile``, for saving descriptor files that were unparseable for later debugging. --- lib/bridgedb/parse/descriptors.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 03b3f55..63d9bf2 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -14,6 +14,8 @@ from __future__ import print_function import datetime import logging +import os +import shutil from stem.descriptor import extrainfo_descriptor from stem.descriptor import server_descriptor @@ -24,6 +26,41 @@ from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb import safelog +def _copyUnparseableDescriptorFile(filename): + """Save a copy of the bad descriptor file for later debugging. + + If the old filename was ``'descriptors/cached-extrainfo.new'``, then the + name of the copy will be something like + ``'descriptors/2014-11-05-01:57:23_cached-extrainfo.new.unparseable'``. + + :param str filename: The path to the unparseable descriptor file that we + should save a copy of. + :rtype: bool + :returns: ``True`` if a copy of the file was saved successfully, and + ``False`` otherwise. + """ + timestamp = datetime.datetime.now() + timestamp = timestamp.isoformat(sep=chr(0x2d)) + timestamp = timestamp.rsplit('.', 1)[0] + + path, sep, fname = filename.rpartition(os.path.sep) + newfilename = "%s%s%s_%s%sunparseable" % (path, sep, timestamp, + fname, os.path.extsep) + + logging.info(("Unparseable descriptor file '%s' will be copied to '%s' " + "for debugging.") % (filename, newfilename)) + + try: + shutil.copyfile(filename, newfilename) + except Exception as error: # pragma: no cover + logging.error(("Could not save copy of unparseable descriptor file " + "in '%s': %s") % (newfilename, str(error))) + return False + else: + logging.debug(("Successfully finished saving a copy of an unparseable " + "descriptor file.")) + return True + def parseNetworkStatusFile(filename, validate=True, skipAnnotations=True, descriptorClass=RouterStatusEntryV3): """Parse a file which contains an ``@type bridge-networkstatus`` document.

1 0

[bridgedb/develop] Add unittest for descriptors with identical timestamps.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 87fc4fa82842d4ddefad64170b202315cd82752e Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 09:24:48 2014 +0000 Add unittest for descriptors with identical timestamps. --- lib/bridgedb/test/test_parse_descriptors.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 4c10874..5ed6f2f 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -303,6 +303,16 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + def test_parse_descriptors_deduplicate_identical_timestamps(self): + """Parsing two descriptors for the same bridge with identical + timestamps should raise a ``b.p.descriptors.DescriptorWarning``. + """ + descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) + descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) + self.assertRaises(descriptors.DescriptorWarning, + descriptors.parseBridgeExtraInfoFiles, + descFileOne, descFileTwo) + def test_parse_descriptors_parseBridgeExtraInfoFiles_two_files(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with two bridge extrainfo files, and check that only the newest extrainfo

1 0

[bridgedb/develop] Fix unittest; discover how many 'transport' lines parsers should find.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 965393fb230cdeb3c0d0d2918af08e8e8417ae73 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 5 02:53:18 2014 +0000 Fix unittest; discover how many 'transport' lines parsers should find. * CHANGE unittest ``test_parse_descriptors_parseBridgeExtraInfoFiles_one_file`` to automatically discover how many transport lines the descriptor parsers should have found. --- lib/bridgedb/test/test_parse_descriptors.py | 30 ++++++++++++--------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index c3772c7..6479168 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -21,7 +21,7 @@ HAS_STEM = False try: from stem.descriptor.server_descriptor import RelayDescriptor - from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor + from stem.descriptor.extrainfo_descriptor import BridgeExtraInfoDescriptor from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb.parse import descriptors except (ImportError, NameError), error: @@ -150,11 +150,6 @@ class ParseDescriptorsTests(unittest.TestCase): def setUp(self): """Test if we have Stem installed. Skip these tests if it's missing.""" - self.expectedIPBridge0 = '2.215.61.223' - self.expectedIPBridge1 = '80.44.173.87' - - self.expectedFprBridge0 = 'E08B324D20AD0A13E114F027AB9AC3F32CA696A0' - if self.skip: raise unittest.SkipTest("Couldn't import Stem.") @@ -182,8 +177,9 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertIsInstance(routers, list) bridge = routers[0] self.assertIsInstance(bridge, RelayDescriptor) - self.assertEqual(bridge.address, self.expectedIPBridge0) - self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) + self.assertEqual(bridge.address, u'152.78.9.20') + self.assertEqual(bridge.fingerprint, + u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') def test_parse_descriptors_parseBridgeNetworkStatusFile_return_type(self): """``b.p.descriptors.parseNetworkStatusFile`` should return a dict.""" @@ -217,15 +213,14 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] - self.assertEqual(bridge.address, self.expectedIPBridge0) - self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) + self.assertEqual(bridge.address, u'152.78.9.20') + self.assertEqual(bridge.fingerprint, + u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') def test_parse_descriptors_parseBridgeNetworkStatusFile_2(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with two bridge networkstatus descriptors. """ - expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] - # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -233,9 +228,9 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_1) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] - - self.assertIn(bridge.address, expectedIPs) - self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) + self.assertEqual(bridge.address, u'152.78.9.20') + self.assertEqual(bridge.fingerprint, + u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') def test_parse_descriptors_parseBridgeExtraInfoFiles_return_type(self): """The return type of ``b.p.descriptors.parseBridgeExtraInfoFiles`` @@ -252,7 +247,7 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] - self.assertIsInstance(bridge, RelayExtraInfoDescriptor) + self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) def test_parse_descriptors_parseBridgeExtraInfoFiles_one_file(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with only one @@ -267,7 +262,8 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(len(bridge.transport), BRIDGE_EXTRA_INFO_DESCRIPTOR.count('transport ')) - self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) + self.assertEqual(bridge.fingerprint, + u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') def test_parse_descriptors_parseBridgeExtraInfoFiles_two_files(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with two

1 0

[bridgedb/develop] Add unittest for parsing extrainfo files without validation.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 7af8ed0d62b60ffc9a066ba64980deb2f3036af3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 09:31:17 2014 +0000 Add unittest for parsing extrainfo files without validation. --- lib/bridgedb/test/test_parse_descriptors.py | 128 +++++++++++++++++++++++---- 1 file changed, 112 insertions(+), 16 deletions(-) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 5ed6f2f..67bef6a 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -11,6 +11,7 @@ from __future__ import print_function +import datetime import io import os import textwrap @@ -21,7 +22,7 @@ HAS_STEM = False try: from stem.descriptor.server_descriptor import RelayDescriptor - from stem.descriptor.extrainfo_descriptor import BridgeExtraInfoDescriptor + from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor from stem.descriptor.router_status_entry import RouterStatusEntryV3 from bridgedb.parse import descriptors except (ImportError, NameError), error: @@ -142,6 +143,37 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' +BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = '''\ +extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 +published 2014-12-04 03:10:25 +write-history 2014-12-04 03:10:25 (900 s) 3188736,2226176,2866176,2226176 +read-history 2014-12-04 03:10:25 (900 s) 3891200,2483200,2698240,2483200 +dirreq-write-history 2014-12-04 03:10:25 (900 s) 1024,0,2048,3072 +dirreq-read-history 2014-12-04 03:10:25 (900 s) 0,0,0,0 +geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F +geoip6-db-digest E983833985E4BCA34CEF611B2DF51942D188E638 +dirreq-stats-end 2014-12-04 03:10:25 (86400 s) +dirreq-v3-ips +dirreq-v3-reqs +dirreq-v3-resp ok=16,not-enough-sigs=0,unavailable=0,not-found=0,not-modified=0,busy=0 +dirreq-v3-direct-dl complete=0,timeout=0,running=0 +dirreq-v3-tunneled-dl complete=12,timeout=0,running=0 +transport obfs3 2.215.61.223:4057 +transport obfs2 2.215.61.223:4058 +transport scramblesuit 2.215.61.223:4059 password=ABCDEFGHIJKLMNOPQRSTUVWXYZ234567 +transport obfs4 2.215.61.223:4060 iat-mode=0,node-id=19a448c01aa2e7d55979473b647e282459995b85,public-key=7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25 +bridge-stats-end 2014-12-04 03:10:25 (86400 s) +bridge-ips ca=8 +bridge-ip-versions v4=8,v6=0 +bridge-ip-transports <OR>=8 +router-signature +-----BEGIN SIGNATURE----- +KOXNPCoe+Q+thFA/Lz7RTja2tWp4oC6SvyIooEZibHtEDgiXuU4sELWT4bSOk3np +RVmu7QPMmNybx4LHowq3pOeNLtJzpWg8Pfo+N6tR+K4nqPwBRmpsuDhCD/tIXJlP +U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= +-----END SIGNATURE----- +''' + class ParseDescriptorsTests(unittest.TestCase): """Unittests for :class:`bridgedb.parse.descriptors` module.""" @@ -150,6 +182,11 @@ class ParseDescriptorsTests(unittest.TestCase): def setUp(self): """Test if we have Stem installed. Skip these tests if it's missing.""" + self.expectedIPBridge0 = '2.215.61.223' + self.expectedIPBridge1 = '80.44.173.87' + + self.expectedFprBridge0 = 'E08B324D20AD0A13E114F027AB9AC3F32CA696A0' + if self.skip: raise unittest.SkipTest("Couldn't import Stem.") @@ -177,9 +214,8 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertIsInstance(routers, list) bridge = routers[0] self.assertIsInstance(bridge, RelayDescriptor) - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + self.assertEqual(bridge.address, self.expectedIPBridge0) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeNetworkStatusFile_return_type(self): """``b.p.descriptors.parseNetworkStatusFile`` should return a dict.""" @@ -203,7 +239,7 @@ class ParseDescriptorsTests(unittest.TestCase): bridge = routers[0] self.assertIsInstance(bridge, RouterStatusEntryV3) - def test_parse_descriptors_parseBridgeNetworkStatusFile_1(self): + def test_parse_descriptors_parseBridgeNetworkStatusFile_one_file(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with one bridge networkstatus descriptor. """ @@ -213,14 +249,15 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + self.assertEqual(bridge.address, self.expectedIPBridge0) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) - def test_parse_descriptors_parseBridgeNetworkStatusFile_2(self): + def test_parse_descriptors_parseBridgeNetworkStatusFile_two_files(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with two bridge networkstatus descriptors. """ + expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] + # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -228,9 +265,9 @@ class ParseDescriptorsTests(unittest.TestCase): BRIDGE_NETWORKSTATUS_1) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] - self.assertEqual(bridge.address, u'152.78.9.20') - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + + self.assertIn(bridge.address, expectedIPs) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_parseBridgeNetworkStatusFile_with_annotations(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with some document @@ -285,7 +322,7 @@ class ParseDescriptorsTests(unittest.TestCase): descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseBridgeExtraInfoFiles(descFile) bridge = routers.values()[0] - self.assertIsInstance(bridge, BridgeExtraInfoDescriptor) + self.assertIsInstance(bridge, RelayExtraInfoDescriptor) def test_parse_descriptors_parseBridgeExtraInfoFiles_one_file(self): """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with only one @@ -300,8 +337,7 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(len(bridge.transport), BRIDGE_EXTRA_INFO_DESCRIPTOR.count('transport ')) - self.assertEqual(bridge.fingerprint, - u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) def test_parse_descriptors_deduplicate_identical_timestamps(self): """Parsing two descriptors for the same bridge with identical @@ -321,5 +357,65 @@ class ParseDescriptorsTests(unittest.TestCase): descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE) routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, descFileTwo) + + # We shouldn't have duplicates: + self.assertEqual(len(routers), 1, + "We shouldn't have any duplicate descriptors.") + + # We should only have the newest descriptor: + bridge = routers.values()[0] + self.assertEqual( + bridge.published, + datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"), + "We should have the newest available descriptor for this router.") + + def test_parse_descriptors_parseBridgeExtraInfoFiles_two_files_reverse(self): + """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with two bridge + extrainfo files. This time, they are processed in reverse to ensure + that we only keep the newer duplicates of descriptors, no matter what + order they appeared in the files. + """ + descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE) + descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) + routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, descFileTwo) + + self.assertEqual(len(routers), 1, + "We shouldn't have any duplicate descriptors.") + + bridge = routers.values()[0] + self.assertEqual( + bridge.published, + datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"), + "We should have the newest available descriptor for this router.") + + def test_parse_descriptors_parseBridgeExtraInfoFiles_three_files(self): + """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with three + bridge extrainfo files, and check that only the newest extrainfo + descriptor is used. + """ + descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE) + descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) + descFileThree = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) + routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, + descFileTwo, + descFileThree) + # We shouldn't have duplicates: - self.assertEqual(len(routers), 1) + self.assertEqual(len(routers), 1, + "We shouldn't have any duplicate descriptors.") + + # We should only have the newest descriptor: + bridge = routers.values()[0] + self.assertEqual( + bridge.published, + datetime.datetime.strptime("2014-12-04 03:10:25", "%Y-%m-%d %H:%M:%S"), + "We should have the newest available descriptor for this router.") + + def test_parse_descriptors_parseBridgeExtraInfoFiles_no_validate(self): + """Test for ``b.p.descriptors.parseBridgeExtraInfoFiles`` with + descriptor validation disabled. + """ + descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) + routers = descriptors.parseBridgeExtraInfoFiles(descFileOne, + validate=False) + self.assertGreaterEqual(len(routers), 1)

1 0

[bridgedb/develop] Add two unittests for parsing networkstatus files with headers.
by isis＠torproject.org 19 Feb '15

19 Feb '15

commit 8b7649bb60b5e4902f74074060679622cd445884 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 09:23:49 2014 +0000 Add two unittests for parsing networkstatus files with headers. --- lib/bridgedb/test/test_parse_descriptors.py | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index 6479168..4c10874 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -232,6 +232,44 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(bridge.fingerprint, u'6FA9216CF3A06E89A03121ACC31F70F8DFD7DDCC') + def test_parse_descriptors_parseBridgeNetworkStatusFile_with_annotations(self): + """Test ``b.p.descriptors.parseNetworkStatusFile`` with some document + headers before the first 'r'-line. + """ + expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] + descFile = 'networkstatus-bridges' + + with open(descFile, 'w') as fh: + fh.write('signature and stuff from the BridgeAuth would go here\n') + fh.write('some more annotations with parameters and stuff\n') + fh.write(BRIDGE_NETWORKSTATUS_0) + fh.write(BRIDGE_NETWORKSTATUS_1) + fh.flush() + + routers = descriptors.parseNetworkStatusFile(descFile) + bridge = routers[0] + self.assertIn(bridge.address, expectedIPs) + self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) + + def test_parse_descriptors_parseBridgeNetworkStatusFile_with_annotations_no_skipping(self): + """Test ``b.p.descriptors.parseNetworkStatusFile`` with some + document headers before the first 'r'-line, but without skipping said + annotations. + """ + expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] + descFile = 'networkstatus-bridges' + + with open(descFile, 'w') as fh: + fh.write('signature and stuff from the BridgeAuth would go here\n') + fh.write('some more annotations with parameters and stuff\n') + fh.write(BRIDGE_NETWORKSTATUS_0) + fh.write(BRIDGE_NETWORKSTATUS_1) + fh.flush() + + self.assertRaises(ValueError, + descriptors.parseNetworkStatusFile, + descFile, skipAnnotations=False) + def test_parse_descriptors_parseBridgeExtraInfoFiles_return_type(self): """The return type of ``b.p.descriptors.parseBridgeExtraInfoFiles`` should be a dictionary (after deduplication).

1 0