January 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Deepcopy the HMAC factory function's resulting function.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit a49ee2a0872b474e11a5f8dd9c169b2fac1ae22e Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:29:38 2013 +0000 Deepcopy the HMAC factory function's resulting function. --- lib/bridgedb/Bridges.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 723d3c9..e8ca1d9 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -7,6 +7,7 @@ them in rings. import binascii import bisect +import copy import hmac import logging import re @@ -79,7 +80,7 @@ def get_hmac_fn(k, hex=True): If 'hex' is true, the output of the function will be hex-encoded.""" h = hmac.new(k, digestmod=DIGESTMOD) def hmac_fn(v): - h_tmp = h.copy() + h_tmp = copy.deepcopy(h) h_tmp.update(v) if hex: return h_tmp.hexdigest()

1 0

[bridgedb/master] Add newly refactored networkstatus parsers and unittests for them.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit a6d52fe28997d1c81c8f8533277925424b9fc157 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:30:38 2013 +0000 Add newly refactored networkstatus parsers and unittests for them. --- lib/bridgedb/parse/networkstatus.py | 236 +++++++++++++++++++++++++ lib/bridgedb/test/test_parse_networkstatus.py | 92 ++++++++++ 2 files changed, 328 insertions(+) diff --git a/lib/bridgedb/parse/networkstatus.py b/lib/bridgedb/parse/networkstatus.py new file mode 100644 index 0000000..d38257d --- /dev/null +++ b/lib/bridgedb/parse/networkstatus.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013 Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +"""Parsers for ``@type bridge-network-status 1.0`` descriptors. + +.. _descriptors: https://metrics.torproject.org/formats.html#descriptortypes + +**Module Overview:** + +.. + parse + \_networkstatus + |_ parseRLine - Parse an 'r'-line from a networkstatus document + |_ parseALine - Parse an 'a'-line from a networkstatus document + \_ parseSLine - Parse an 's'-line from a networkstatus document +""" + +import binascii +import logging +import string +import time + +from bridgedb.parse import addr +from bridgedb.parse import padBase64 + + +class NetworkstatusParsingError(Exception): + """Unable to parse networkstatus document line.""" + +class InvalidNetworkstatusRouterIdentity(ValueError): + """The ID field of a networkstatus document 'r'-line is invalid.""" + +class InvalidNetworkstatusDescriptorDigest(ValueError): + """Descriptor digest of a networkstatus document 'r'-line is invalid.""" + + +def isValidRouterNickname(nickname): + """Determine if a router's given nickname meets the specification. + + :param string nickname: An OR's nickname. + """ + + +def parseRLine(line): + """Parse an 'r'-line from a networkstatus document. + + From torspec.git/dir-spec.txt, commit 36761c7d553d L1499-1512: + | + |"r" SP nickname SP identity SP digest SP publication SP IP SP ORPort + | SP DirPort NL + | + | [At start, exactly once.] + | + | "Nickname" is the OR's nickname. "Identity" is a hash of its + | identity key, encoded in base64, with trailing equals sign(s) + | removed. "Digest" is a hash of its most recent descriptor as + | signed (that is, not including the signature), encoded in base64. + | "Publication" is the + | publication time of its most recent descriptor, in the form + | YYYY-MM-DD HH:MM:SS, in UTC. "IP" is its current IP address; + | ORPort is its current OR port, "DirPort" is its current directory + | port, or "0" for "none". + | + + :param string line: An 'r'-line from an bridge-network-status descriptor. + + """ + (nickname, ID, descDigest, timestamp, + ORaddr, ORport, dirport) = (None for x in xrange(7)) + + if not line.startswith('r '): + raise NetworkstatusParsingError( + "Networkstatus parser received non 'r'-line: %r" % line) + + line = line[2:] # Chop of the 'r ' + + fields = line.split() + if len(fields) != 8: + raise NetworkstatusParsingError( + "Wrong number of fields in networkstatus 'r'-line: %r" % line) + + try: + nickname, ID = fields[:2] + + if ID.endswith('='): + raise InvalidNetworkstatusRouterIdentity( + "Skipping networkstatus parsing for router with nickname %r: ", + "Unpadded, base64-encoded networkstatus router identity ", + "string ends with '=': %r" % (nickname, ID)) + try: + ID = padBase64(ID) # Add the trailing equals sign back in + except (AttributeError, ValueError) as error: + raise InvalidNetworkstatusRouterIdentity(error.message) + + ID = binascii.a2b_base64(ID) + if not ID: + raise InvalidNetworkstatusRouterIdentity( + "Skipping networkstatus parsing for router with nickname %r: ", + "Base64-encoding for networkstatus router identity string is ", + "invalid! Line: %r" % (nickname, line)) + + except IndexError as error: + logging.error(error.message) + except InvalidNetworkstatusRouterIdentity as error: + logging.error(error.message) + ID = None + + try: + descDigest = binascii.a2b_base64(fields[2]) + except (AttributeError, ValueError) as error: + raise InvalidNetworkstatusDescriptorDigest(error.message) + + + timestamp = time.mktime(time.strptime(" ".join(fields[3:5]), + "%Y-%m-%d %H:%M:%S")) + ORaddr = fields[5] + ORport = fields[6] + dirport = fields[7] + + finally: + return (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport) + +def parseALine(line, fingerprint=None): + """Parse an 'a'-line of a bridge networkstatus document. + + From torspec.git/dir-spec.txt, commit 36761c7d553d L1499-1512: + | + | "a" SP address ":" port NL + | + | [Any number.] + | + | Present only if the OR has at least one IPv6 address. + | + | Address and portlist are as for "or-address" as specified in + | 2.1. + | + | (Only included when the vote or consensus is generated with + | consensus-method 14 or later.) + + :param string line: An 'a'-line from an bridge-network-status descriptor. + :raises: :exc:`NetworkstatusParsingError` + :rtype: tuple + :returns: A 2-tuple of a string respresenting the IP address and a + :class:`bridgedb.parse.addr.PortList`. + """ + ip = None + address = None + portlist = None + + if not line.startswith('a '): + logging.error("Networkstatus parser received non 'a'-line for %r:" + % (fingerprint or 'Unknown')) + logging.error("\t%r" % line) + return address, portlist + + line = line[2:] # Chop off the 'a ' + + try: + ip, portlist = line.rsplit(':', 1) + except (IndexError, ValueError, addr.InvalidPort) as error: + logging.exception(error) + raise NetworkstatusParsingError( + "Parsing networkstatus 'a'-line for %r failed! Line: %r" + %(fingerprint, line)) + else: + ip = ip.strip('[]') + address = addr.isIPAddress(ip) + if not ip: + raise NetworkstatusParsingError( + "Got invalid IP Address in networkstatus 'a'-line for %r: %r" + % (fingerprint, ip)) + + portlist = addr.PortList(portlist) + + logging.debug("Parsed networkstatus ORAddress line for %r:" % fingerprint) + logging.debug("\tAddress: %s \tPorts: %s" % (address, portlist)) + + return address, portlist + +def parseSLine(line): + """Parse an 's'-line from a bridge networkstatus document. + + The 's'-line contains all flags assigned to a bridge. The flags which may + be assigned to a bridge are as follows: + + From torspec.git/dir-spec.txt, commit 36761c7d553d L1526-1554: + | + | "s" SP Flags NL + | + | [Exactly once.] + | + | A series of space-separated status flags, in lexical order (as ASCII + | byte strings). Currently documented flags are: + | + | "BadDirectory" if the router is believed to be useless as a + | directory cache (because its directory port isn't working, + | its bandwidth is always throttled, or for some similar + | reason). + | "Fast" if the router is suitable for high-bandwidth circuits. + | "Guard" if the router is suitable for use as an entry guard. + | "HSDir" if the router is considered a v2 hidden service directory. + | "Named" if the router's identity-nickname mapping is canonical, + | and this authority binds names. + | "Stable" if the router is suitable for long-lived circuits. + | "Running" if the router is currently usable. + | "Valid" if the router has been 'validated'. + | "V2Dir" if the router implements the v2 directory protocol. + + :param string line: An 's'-line from an bridge-network-status descriptor. + :rtype: tuple + :returns: A 2-tuple of booleans, the first is True if the bridge has the + "Running" flag, and the second is True if it has the "Stable" flag. + """ + fast, running, stable, guard, valid = False + + line = line[2:] + + flags = [x.capitalize() for x in line.split()] + fast = 'Fast' in flags + running = 'Running' in flags + stable = 'Stable' in flags + guard = 'Guard' in flags + valid = 'Valid' in flags + + logging.debug("Parsed Flags: %s" % flags) + + # Right now, we only care about 'Running' and 'Stable' + return running, stable diff --git a/lib/bridgedb/test/test_parse_networkstatus.py b/lib/bridgedb/test/test_parse_networkstatus.py new file mode 100644 index 0000000..830f96c --- /dev/null +++ b/lib/bridgedb/test/test_parse_networkstatus.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013, Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-Clause BSD, see LICENSE for licensing information + +"""Unittests for the :mod:`bridgedb.parse.networkstatus` module. + +These tests are meant to ensure that the :mod:`bridgedb.parse.networkstatus` +module is functioning correctly. +""" + +from __future__ import print_function + +from twisted.trial import unittest +from bridgedb.parse import networkstatus + +import sure +from sure import this, these, those, the, it + + +class ParseNetworkStatusRLineTests(unittest.TestCase): + """Tests for :func:`bridgedb.parse.networkstatus.parseRLine`.""" + + pre = 'r ' + nick = 'Testing' + ident = 'bXw2N1K9AAKR5undPaTgNUySNxI' + desc = 'Z6cisoPT9s6hEd4JkHFAlIWAwXQ=' + ts = '2013-10-31 15:15:15' + ip = '221.251.0.42' + port = '9001' + dirp = '0' + + def test_missingPrefix(self): + line = ' '.join([self.nick, self.ident, self.desc, + self.ts, self.ip, self.port, self.dirp]) + self.assertRaises(networkstatus.NetworkstatusParsingError, + networkstatus.parseRLine, line) + + def test_wrongNumberOfFields(self): + line = ' '.join([self.pre, self.nick, self.ident, self.ts, self.ip]) + self.assertRaises(networkstatus.NetworkstatusParsingError, + networkstatus.parseRLine, line) + + def test_wrongFieldOrder(self): + line = ' '.join([self.pre, self.nick, self.desc, self.ident, + self.ts, self.ip, self.port, self.dirp]) + fields = networkstatus.parseRLine(line) + nick, others = fields[0], fields[1:] + + this(nick).should.be.ok + this(nick).should.be.a(str) + this(nick).should.equal(self.nick) + + the(others).should.be.a(tuple) + the(others).should.have.length_of(6) + for other in others: + the(other).should.be(None) + + def test_invalidTimestampMissingDate(self): + line = ' '.join([self.pre, self.nick, self.ident, self.desc, + '15:15:15', self.ip, self.port, self.dirp]) + self.assertRaises(networkstatus.NetworkstatusParsingError, + networkstatus.parseRLine, line) + + def test_invalidBase64(self): + line = ' '.join([self.pre, self.nick, '%$>#@,<', self.desc, + self.ts, self.ip, self.port, self.dirp]) + nick, ident, desc, ts, ip, port, dirp = networkstatus.parseRLine(line) + + the(nick).should.be.ok + the(nick).should.be.a(str) + the(nick).should.equal(self.nick) + + the(ident).should.be(None) + the(desc).should.be(None) + + def test_invalidTimestamp(self): + line = ' '.join([self.pre, self.nick, self.ident, self.desc, + '123456789 987654321', self.ip, self.port, self.dirp]) + fields = networkstatus.parseRLine(line) + + def test_invalidIPAddress(self): + line = ' '.join([self.pre, self.nick, self.ident, self.desc, + self.ts, '0.0.0.0', self.port, self.dirp]) + fields = networkstatus.parseRLine(line) +

1 0

[bridgedb/master] Slight PEP8ification to bridgedb.Bridges.IPBasedDistributor.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit e73bd6cc744978e3f51b9afd478c74b62d52f053 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:34:58 2013 +0000 Slight PEP8ification to bridgedb.Bridges.IPBasedDistributor. --- lib/bridgedb/Dist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index e3481f0..0b1a149 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -104,8 +104,8 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): # add splitter and cache the default rings # plus leave room for dynamic filters ring_cache_size = self.nClusters + len(ipCategories) + 5 - self.splitter = bridgedb.Bridges.FilteredBridgeSplitter(key2, - max_cached_rings=ring_cache_size) + self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( + key2, max_cached_rings=ring_cache_size) logging.debug("added splitter %s" % self.splitter)

1 0

[bridgedb/master] Add an XXX about the odd, hardcoded number of "extra hashrings".
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 961918843eea02d0ff7d64e1c0a436f55f9ac33e Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:37:55 2013 +0000 Add an XXX about the odd, hardcoded number of "extra hashrings". --- lib/bridgedb/Dist.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 66ca438..f599e47 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -103,6 +103,9 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): # add splitter and cache the default rings # plus leave room for dynamic filters + # + # XXX Why is the "extra room" hardcoded to be 5? Shouldn't it be some + # fraction of the number of clusters/categories? --isis ring_cache_size = self.nClusters + len(ipCategories) + 5 self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( key2, max_cached_rings=ring_cache_size)

1 0

[bridgedb/master] Make the logs clearly state which distributor bridges are being inserted into.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit c6230c237f161d67e0002a80d98c41f73dfced0e Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:36:51 2013 +0000 Make the logs clearly state which distributor bridges are being inserted into. --- lib/bridgedb/Dist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 0b1a149..66ca438 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -106,8 +106,8 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): ring_cache_size = self.nClusters + len(ipCategories) + 5 self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( key2, max_cached_rings=ring_cache_size) - - logging.debug("added splitter %s" % self.splitter) + logging.debug("Added splitter %s to IPBasedDistributor." + % self.splitter.__class__) def prepopulateRings(self): # populate all rings (for dumping assignments and testing)

1 0

[bridgedb/master] Refactor Bridges.parseStatusFile() to use bridgedb.parse.networkstatus.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit c829b8f4c2e9f1b71d500b39d8e7713f1d66fa17 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:31:45 2013 +0000 Refactor Bridges.parseStatusFile() to use bridgedb.parse.networkstatus. --- lib/bridgedb/Bridges.py | 75 ++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index e8ca1d9..6903ce6 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -20,6 +20,9 @@ import random import bridgedb.Storage import bridgedb.Bucket +from bridgedb.parse import networkstatus + + HEX_FP_LEN = 40 ID_LEN = 20 @@ -483,59 +486,51 @@ def parseExtraInfoFile(f): if ID and line.startswith("router-signature"): ID = None -def parseStatusFile(f): - """DOCDOC""" - timestamp = ID = None - num_or_address_lines = 0 +def parseStatusFile(networkstatusFile): + """Parse entries in a bridge networkstatus file. + + :type networkstatusFile: A file-like object. + :param networkstatusFile: A file containing `@type bridge-networkstatus` documents. + """ + (nickname, ID, descDigest, timestamp, + ORaddr, ORport, dirport, addr, portlist) = (None for x in xrange(9)) + running = stable = False + parsedORAddressLines = 0 or_addresses = {} - for line in f: - if not ID: - logging.debug("Parsing network status line") + + for line in networkstatusFile: line = line.strip() if line.startswith("opt "): line = line[4:] if line.startswith("r "): - try: - ID = binascii.a2b_base64(line.split()[2]+"=") - timestamp = time.mktime(time.strptime( - " ".join(line.split()[4:6]), "%Y-%m-%d %H:%M:%S") - ) - logging.debug(" Fingerprint: %s", toHex(ID)) - logging.debug(" Timestamp: %s", timestamp) - except binascii.Error: - logging.warn(" Unparseable base64 ID %r", line.split()[2]) - except ValueError: - timestamp = None - logging.debug(" Timestamp; Invalid") + (nickname, ID, descDigest, timestamp, + ORaddr, ORport, dirport) = networkstatus.parseRLine(line) elif ID and line.startswith("a "): - if num_or_address_lines < 8: - line = line[2:] - try: - address,portlist = parseORAddressLine(line) - logging.debug(" Parsed address: %s", address) - logging.debug(" Parsed port(s): %s", portlist) - except ParseORAddressError: - logging.debug(" Failed to Parsed address: %s", address) - try: - or_addresses[address].add(portlist) - except KeyError: - or_addresses[address] = portlist + try: + addr, portlist = networkstatus.parseALine(line, toHex(ID)) + except networkstatus.ParseNetstatusError as error: + logging.error(error.message) else: - logging.warn(" Skipping extra or-address line "\ - "from Bridge with ID %r" % ID) - num_or_address_lines += 1 + + try: or_addresses[addr].add(portlist) + except KeyError: or_addresses[addr] = portlist + parsedORAddressLines += 1 elif ID and timestamp and line.startswith("s "): - flags = line.split() - logging.debug(" Parsed Flags: %s", flags) - yield ID, ("Running" in flags), ("Stable" in flags), or_addresses, timestamp - timestamp = ID = None - logging.debug(" Total: %d OR address lines", num_or_address_lines) - num_or_address_lines = 0 + running, stable = networkstatus.parseSLine(line) + + yield ID, running, stable, or_addresses, timestamp + + (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport, + addr, portlist) = (None for x in xrange(9)) + running = stable = False or_addresses = {} + logging.debug("Total ORAddress lines parsed from '%s': %d" + % (networkstatusFile.name, parsedORAddressLines)) + def parseCountryBlockFile(f): """Generator. Parses a blocked-bridges file 'f', and yields a fingerprint (ID), address, a list of ports, and a list of country

1 0

[bridgedb/master] Use a loaded persistent.State in Main.load().
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 11a581a816927e441582e361c19234b1b5c7b746 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 16:11:26 2013 +0000 Use a loaded persistent.State in Main.load(). * ADD a docstring to Main.load(). --- lib/bridgedb/Main.py | 121 +++++++++++++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 46 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 8ef3fee..3729a17 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -68,40 +68,59 @@ def configureLogging(cfg): else: logging.warn("Safe Logging: Disabled") - -def load(cfg, splitter, clear=False): - """Read all the bridge files from cfg, and pass them into a splitter - object. +def load(state, splitter, clear=False): + """Read and parse all descriptors, and load into a bridge splitter. + + Read all the appropriate bridge files from the saved + :class:`~bridgedb.persistent.State`, parse and validate them, and then + store them into our ``state.splitter`` instance. The ``state`` will be + saved again at the end of this function. + + :type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>` + :param splitter: A class which provides a mechanism for HMACing + Bridges in order to assign them to hashrings. + :param boolean clear: If True, clear all previous bridges from the + splitter before parsing for new ones. """ + if not state: + logging.fatal("bridgedb.Main.load() could not retrieve state!") + sys.exit(2) + if clear: - logging.info("Clearing old bridges") + logging.info("Clearing old bridges...") splitter.clear() - logging.info("Loading bridges") + + logging.info("Loading bridges...") + status = {} addresses = {} timestamps = {} - if hasattr(cfg, "STATUS_FILE"): - logging.info("Opening Network Status document %s" % cfg.STATUS_FILE) - f = open(cfg.STATUS_FILE, 'r') - for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(f): - status[ID] = running, stable - addresses[ID] = or_addresses - if ID in timestamps.keys(): timestamps[ID].append(timestamp) - else: timestamps[ID] = [timestamp] - #transports[ID] = transports - logging.debug("Closing status document") - f.close() - bridges = {} + + logging.info("Opening network status file: %s" % state.STATUS_FILE) + f = open(state.STATUS_FILE, 'r') + for (ID, running, stable, + or_addresses, timestamp) in Bridges.parseStatusFile(f): + + status[ID] = running, stable + addresses[ID] = or_addresses + + if ID in timestamps.keys(): + timestamps[ID].append(timestamp) + else: + timestamps[ID] = [timestamp] + #transports[ID] = transports + logging.debug("Closing network status file") + f.close() + + bridges = {} db = bridgedb.Storage.getDB() - for fname in cfg.BRIDGE_FILES: - logging.info("Opening cached server-descriptor document: '%s'" % fname) - logging.debug("Parsing document for purpose=%s" % cfg.BRIDGE_PURPOSE) + for fname in state.BRIDGE_FILES: + logging.info("Opening bridge-server-descriptor file: '%s'" % fname) f = open(fname, 'r') - for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): + for bridge in Bridges.parseDescFile(f, state.BRIDGE_PURPOSE): if bridge.getID() in bridges: - logging.warn( - "Parsed bridge that we've already added. Skipping.") + logging.warn("Parsed a duplicate bridge. Skipping.") continue else: bridges[bridge.getID()] = bridge @@ -109,63 +128,73 @@ def load(cfg, splitter, clear=False): if s is not None: running, stable = s bridge.setStatus(running=running, stable=stable) + # XXX: what do we do with all these or_addresses? + # + # The bridge stability metrics are only concerned with a + # single ip:port So for now, we will only consider the bridges + # primary IP:port bridge.or_addresses = addresses.get(bridge.getID()) splitter.insert(bridge) - # add or update BridgeHistory entries into the database - # XXX: what do we do with all these or_addresses? - # The bridge stability metrics are only concerned with a single ip:port - # So for now, we will only consider the bridges primary IP:port + if bridge.getID() in timestamps.keys(): ts = timestamps[bridge.getID()][:] ts.sort() for timestamp in ts: + logging.debug( + "Adding/updating timestamps in BridgeHistory for ", + "'%s' in database: %s" + % (bridge.getID(), timestamp)) bridgedb.Stability.addOrUpdateBridgeHistory( bridge, timestamp) - logging.debug("Closing server-descriptor document") + logging.debug("Closing bridge-server-descriptor file: '%s'" % fname) f.close() # read pluggable transports from extra-info document # XXX: should read from networkstatus after bridge-authority # does a reachability test - for filename in cfg.EXTRA_INFO_FILES: - logging.info("Opening extra-info document: '%s'" % filename) + for filename in state.EXTRA_INFO_FILES: + logging.info("Opening extra-info file: '%s'" % filename) f = open(filename, 'r') for transport in Bridges.parseExtraInfoFile(f): ID, method_name, address, port, argdict = transport try: if bridges[ID].running: - logging.debug(" Appending transport to running bridge") + logging.debug("Adding %s transport to running bridge" + % method_name) bridgePT = Bridges.PluggableTransport( bridges[ID], method_name, address, port, argdict) bridges[ID].transports.append(bridgePT) if not bridgePT in bridges[ID].transports: - logging.critical("""Added transport...it disappeared! - Transport: %r""" % bridgePT) + logging.critical( + "Added a transport, but it disappeared!", + "\tTransport: %r" % bridgePT) except KeyError as error: logging.error("Could not find bridge with fingerprint '%s'." % Bridges.toHex(ID)) - logging.debug("Closing extra-info document") + logging.debug("Closing extra-info file: '%s'" % filename) f.close() - if hasattr(cfg, "COUNTRY_BLOCK_FILE"): + + if state.COUNTRY_BLOCK_FILE: logging.info("Opening Blocking Countries file %s" - % cfg.COUNTRY_BLOCK_FILE) - f = open(cfg.COUNTRY_BLOCK_FILE, 'r') - for ID, address, portlist, countries in Bridges.parseCountryBlockFile(f): + % state.COUNTRY_BLOCK_FILE) + f = open(state.COUNTRY_BLOCK_FILE) + # Identity digest, primary OR address, portlist, country codes + for ID, addr, portlist, cc in Bridges.parseCountryBlockFile(f): if ID in bridges.keys() and bridges[ID].running: for port in portlist: - logging.debug(":.( Tears! %s blocked %s %s:%s" - % (countries, bridges[ID].fingerprint, - address, port)) + addrport = "{0}:{1}".format(addr, port) + logging.debug(":'( Tears! %s blocked bridge %s at %s" + % (cc, bridges[ID].fingerprint, addrport)) try: - bridges[ID].blockingCountries["%s:%s" % \ - (address, port)].update(countries) + bridges[ID].blockingCountries[addrport].update(cc) except KeyError: - bridges[ID].blockingCountries["%s:%s" % \ - (address, port)] = set(countries) + bridges[ID].blockingCountries[addrport] = set(cc) logging.debug("Closing blocking-countries document") f.close() bridges = None + state.save() + return def loadConfig(configFile=None, configCls=None): """Load configuration settings on top of the current settings.

1 0

[bridgedb/master] Add parse.padBase64() function for adding stripped padding '==' back in.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit bc8b04929fccdcf1f7d7f6c162a677cae4ccfa70 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:39:20 2013 +0000 Add parse.padBase64() function for adding stripped padding '==' back in. --- lib/bridgedb/parse/__init__.py | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/lib/bridgedb/parse/__init__.py b/lib/bridgedb/parse/__init__.py index e69de29..1fdbd8a 100644 --- a/lib/bridgedb/parse/__init__.py +++ b/lib/bridgedb/parse/__init__.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013 Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +"""Modules for parsing data. + +** Package Overview: ** + +.. + parse + ||_ parse.addr + | |_ isIPAddress - Check if an arbitrary string is an IP address. + | |_ isIPv4 - Check if an arbitrary string is an IPv4 address. + | |_ isIPv6 - Check if an arbitrary string is an IPv6 address. + | \_ isValidIP - Check that an IP address is valid. + | + |__ :mod:`bridgedbparse.headers` + |__ :mod:`bridgedb.parse.options` + \__ :mod:`bridgedb.parse.versions` +""" + + +def padBase64(b64string): + """Re-add any stripped equals sign character padding to a b64 string. + + :param string b64string: A base64-encoded string which might have had its + trailing equals sign padding removed. + """ + try: + b64string = b64string.strip() + except AttributeError: + logging.error("Cannot pad base64 string %r: not a string." % b64string) + else: + addchars = 0 + remainder = len(b64string) % 4 + if 2 <= remainder <= 3: + addchars = 4 - remainder + else: + raise ValueError("Invalid base64-encoded string: %r" % b64string) + b64string += '=' * addchars + finally: + return b64string

1 0

[bridgedb/master] The identity is based on the public key of the router.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit f9b9ad7cf1dafeb59a54d7e4d3cf922586a3d05e Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sat Nov 9 17:12:45 2013 +0000 The identity is based on the public key of the router. Specifically it is the SHA-1 hash of the DER encoding of an ASN.1 RSA public key. (cherry picked from commit 6b57521a522abaa8f5fdd158708f382293a59e48) Signed-off-by: Isis Lovecruft <isis(a)torproject.org> Conflicts: scripts/gen_bridge_descriptors Matt and I both fixed the same bug, but it turns out we were both a tiny bit wrong, I believe, in different ways: I was improperly PEM-encoding the OR keys, and wasn't using ASN.1 format. Matt was using a dump of SIDPKey as the OR bridge's public identity key -- PyOpenSSL has this rather odd API where you have to dump the public key from the public cert to access it. Also, it's necessary to base64-encode the digest of the identity key, and strip the '=' character base64 padding. We *might* still be doing it wrong and missing the DER-encoding step. --- scripts/gen_bridge_descriptors | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/gen_bridge_descriptors b/scripts/gen_bridge_descriptors index 4d7c930..019e1e3 100644 --- a/scripts/gen_bridge_descriptors +++ b/scripts/gen_bridge_descriptors @@ -98,6 +98,7 @@ OPENSSL_BEGIN_CERT = "-----BEGIN CERTIFICATE-----" OPENSSL_END_CERT = "-----END CERTIFICATE-----" PEM = OpenSSL.crypto.FILETYPE_PEM +ASN1 = OpenSSL.crypto.FILETYPE_ASN1 class OpenSSLKeyGenError(Exception): @@ -573,15 +574,10 @@ def makeOnionKeys(bridge=True, digest='sha1'): onion = createKey() onionSKey, onionSCert, onionPKey, onionPCert = onion - # This is the fingerprint of the server ID key, if we aren't a bridge. If - # we are a bridge, then this is the real fingerprint, which goes into our - # descriptor (but not the one that other ORs see when they connect to us) - fingerprint = CIDPCert.digest(digest) - onionKeyString = 'onion-key\n%s' % getPEMPublicKey(onionPCert) signingKeyString = 'signing-key\n%s' % getPEMPublicKey(signPCert) - return SIDSKey, SIDPCert, (fingerprint, onionKeyString, signingKeyString) + return SIDSKey, SIDPCert, (onionKeyString, signingKeyString) def generateExtraInfo(fingerprint, ts, ipv4, port): """Create an OR extra-info document. @@ -745,17 +741,20 @@ def generateDescriptors(): timestamp = makeTimeStamp(variation=True, period=36) protocols = makeProtocolsLine(vers) - SIDSKey, SIDPCert, (fingerprint, onionkey, signingkey) = makeOnionKeys() + SIDSKey, SIDPCert, (onionkey, signingkey) = makeOnionKeys() idkey_private = getPEMPrivateKey(SIDSKey) idkey_digest = hashlib.sha1(idkey_private).digest() fpr = convertToSpaceyFingerprint(fingerprint) - idkey_public = OpenSSL.crypto.dump_privatekey(PEM, + idkey_public = OpenSSL.crypto.dump_privatekey(ASN1, SIDPCert.get_pubkey()) idkey_public = re.sub(OPENSSL_BEGIN_KEY, '', idkey_public) idkey_public = re.sub(OPENSSL_END_KEY, '', idkey_public) idkey_public = idkey_public.strip() + + ident_digest = hashlib.sha1(idkey_public).digest() + identity = binascii.b2a_base64( hashlib.sha1(idkey_public).digest()).strip().strip('=======')

1 0

[bridgedb/master] Sometimes we want spacey fingerprints
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit ef0dd5ae4b3b551cc6710e2f21691d975b85a924 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sat Nov 9 16:57:33 2013 +0000 Sometimes we want spacey fingerprints Take a string with 40 characters and add a space after every 4 chars. (cherry picked from commit 63c7e9ca9c7bf8d397437d5c87dff8f73a759317) Signed-off-by: Isis Lovecruft <isis(a)torproject.org> --- scripts/gen_bridge_descriptors | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/scripts/gen_bridge_descriptors b/scripts/gen_bridge_descriptors index 68d58c1..4d7c930 100644 --- a/scripts/gen_bridge_descriptors +++ b/scripts/gen_bridge_descriptors @@ -223,37 +223,26 @@ def makeProtocolsLine(version=None): return line def convertToSpaceyFingerprint(fingerprint): - """Convert a colon-delimited fingerprint to be space-delimited. + """Convert to a space-delimited 40 character fingerprint - Given a fingerprint with sets of two hexidecimal characters separated by - colons, for example a certificate or key fingerprint output from OpenSSL: + Given a 40 character string, usually the the SHA-1 hash of the + DER encoding of an ASN.1 RSA public key, such as: | - | 51:58:9E:8B:BF:5C:F6:5A:68:CB:6D:F4:C7:6F:98:C6:B2:02:69:45 + | 72C2F0AE1C14F40ED37ED5F5434B64711A658E46 | convert it to the following format: | - | 5158 9E8B BF5C F65A 68CB 6DF4 C76F 98C6 B202 6945 + | 72C2 F0AE 1C14 F40E D37E D5F5 434B 6471 1A65 8E46 | - :param string fingerprint: A 2-character colon-delimited hex fingerprint. + :param string fingerprint: A 40 character hex fingerprint. :rtype: string :returns: A 4-character space-delimited fingerprint. """ - # The boolean of the non-equality is necessary because -1 is somehow - # truthy in Python… - check = lambda f: bool(f.find(':') != -1) - - while True: - if check(fingerprint): - fingerprint = ''.join(fingerprint.split(':', 1)) - if check(fingerprint): - fingerprint = ' '.join(fingerprint.split(':', 1)) - continue - break - break - - return fingerprint + + assert len(fingerprint) == 40 + return " ".join([fingerprint[i:i+4] for i in xrange(0, 40, 4)]) def makeFingerprintLine(fingerprint, version=None): """Generate an appropriate [bridge-]server-descriptor 'fingerprint' line.

1 0