June 2015 - tor-commits - lists.torproject.org

[bridgedb/develop] Remove unnecessary database initialisation from bridgedb.Main.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit 3bad1b9fcb698ba5d74a07499dcdd23409d5b044 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 11 21:24:07 2015 +0000 Remove unnecessary database initialisation from bridgedb.Main. Conflicts: lib/bridgedb/Main.py --- lib/bridgedb/Main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 1714a53..76c122c 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -378,7 +378,6 @@ def run(options, reactor=reactor): # Initialize our DB. bridgedb.Storage.initializeDBLock() - db = bridgedb.Storage.openDatabase(cfg.DB_FILE + ".sqlite") bridgedb.Storage.setDBFilename(cfg.DB_FILE + ".sqlite") load(state, splitter, clear=False)

1 0

[bridgedb/develop] Remove unused imports from bridgedb.Main module.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit e792fc2c56eae9467dbc30f0489c3f78dd02146b Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 11 21:23:46 2015 +0000 Remove unused imports from bridgedb.Main module. Conflicts: lib/bridgedb/Main.py --- lib/bridgedb/Main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index d6b31a9..1714a53 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -27,19 +27,15 @@ from bridgedb import proxy from bridgedb import safelog from bridgedb import schedule from bridgedb import util -from bridgedb.bridges import InvalidPluggableTransportIP from bridgedb.bridges import MalformedBridgeInfo -from bridgedb.bridges import MalformedPluggableTransport from bridgedb.bridges import MissingServerDescriptorDigest from bridgedb.bridges import ServerDescriptorDigestMismatch from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus from bridgedb.bridges import PluggableTransport from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig -from bridgedb.configure import Conf from bridgedb.parse import descriptors from bridgedb.parse import options -from bridgedb.parse.addr import isIPAddress import bridgedb.Storage

1 0

[bridgedb/develop] Remove integration test monkeypatching for very old, deprecated parsers.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit 452f2aa9c5cfb4f6c80615c89de75f9f45c0d685 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 11 22:42:06 2015 +0000 Remove integration test monkeypatching for very old, deprecated parsers. * REMOVE bridgedb.test.deprecated_networkstatus. * REMOVE descriptor parsers from bridgedb.test.deprecated (which were deprecated by bridgedb.test.deprecated_networkstatus, which was in turn deprecated by the Stem-based parsers in bridgedb.parse.descriptors as part of #9380). * REMOVE monkeypatching of old unittests with (doubly) deprecated parsers. Conflicts: lib/bridgedb/test/deprecated.py lib/bridgedb/test/test_Tests.py --- lib/bridgedb/test/deprecated.py | 254 ----------------------- lib/bridgedb/test/deprecated_networkstatus.py | 275 ------------------------- lib/bridgedb/test/legacy_Tests.py | 1 - lib/bridgedb/test/test_Tests.py | 11 - 4 files changed, 541 deletions(-) diff --git a/lib/bridgedb/test/deprecated.py b/lib/bridgedb/test/deprecated.py index 181814c..df660b6 100644 --- a/lib/bridgedb/test/deprecated.py +++ b/lib/bridgedb/test/deprecated.py @@ -20,260 +20,6 @@ from twisted.python import deprecate from twisted.python.versions import Version -PORTSPEC_LENGTH = 16 - -re_ipv6 = re.compile("\[([a-fA-F0-9:]+)\]:(.*$)") -re_ipv4 = re.compile("((?:\d{1,3}\.?){4}):(.*$)") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 0, 1), - "Removed due to 'bridgedb.Bridges.PortList' being moved to "\ - "'bridgedb.parse.addr.PortList.", - "bridgedb.Bridges", - "PORTSPEC_LENGTH") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 0, 1), - "Attribute 'bridgedb.Bridges.re_ipv4' was removed due to "\ - "'bridgedb.Bridges.parseORAddressLine' moving to "\ - "'bridgedb.parse.networkstatus.", - "bridgedb.Bridges", - "re_ipv4") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 0, 1), - "Attribute 'bridgedb.Bridges.re_ipv6' was removed due to "\ - "'bridgedb.Bridges.parseORAddressLine' moving to "\ - "'bridgedb.parse.networkstatus.", - "bridgedb.Bridges", - "re_ipv6") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 2, 3), - ("Removed due to 'bridgedb.Bridges.HEX_FP_LEN' being moved to " - "'bridgedb.parse.fingerprint.HEX_FINGERPRINT_LEN."), - "bridgedb.Bridges", "HEX_FP_LEN") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 2, 3), - ("Removed due to 'bridgedb.Bridges.toHex' being moved to " - "'bridgedb.parse.fingerprint.toHex."), - "bridgedb.Bridges", "toHex") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 2, 3), - ("Removed due to 'bridgedb.Bridges.fromHex' being moved to " - "'bridgedb.parse.fingerprint.fromHex."), - "bridgedb.Bridges", "fromHex") - - -(a)deprecate.deprecated( - Version('bridgedb', 0, 0, 1), - replacement='bridgedb.parse.networkstatus.parseALine') -def parseORAddressLine(line): - """Deprecated :func:`bridgedb.Bridges.parseORAddressLine`, removed in - bridgedb-0.1.0, in commit 1f111e5. - - This function and the newer parsers from :mod:`bridgedb.parse.netstatus` - are alternately :api:`~twisted.python.monkey.MonkeyPatcher.patch`ed into - the :mod:`old unittests <bridgedb.Tests>`, so that the later functions as - a suite of regression tests. - """ - address = None - portlist = None - # try regexp to discover ip version - for regex in [re_ipv4, re_ipv6]: - m = regex.match(line) - if m: - # get an address and portspec, or raise ParseError - try: - address = ipaddr.IPAddress(m.group(1)) - portlist = PortList(m.group(2)) - except (IndexError, ValueError): raise ParseORAddressError(line) - - # return a valid address, portlist or raise ParseORAddressError - if address and portlist and len(portlist): return address,portlist - raise ParseORAddressError(line) - -(a)deprecate.deprecated(Version('bridgedb', 0, 2, 3), - replacement='bridgedb.parse.fingerprint.isValidFingerprint') -def is_valid_fingerprint(fp): - """Return true iff fp in the right format to be a hex fingerprint - of a Tor server. - """ - if len(fp) != HEX_FP_LEN: - return False - try: - fromHex(fp) - except TypeError: - return False - else: - return True - -(a)deprecate.deprecated(Version('bridgedb', 0, 2, 4), - replacement='bridgedb.parse.descriptors.parseExtraInfoFiles') -def parseExtraInfoFile(f): - """ - parses lines in Bridges extra-info documents. - returns an object whose type corresponds to the - relevant set of extra-info lines. - - presently supported lines and the accompanying type are: - - { 'transport': PluggableTransport, } - - 'transport' lines (torspec.git/proposals/180-pluggable-transport.txt) - - Bridges put the 'transport' lines in their extra-info documents. - the format is: - - transport SP <methodname> SP <address:port> [SP arglist] NL - """ - - ID = None - for line in f: - line = line.strip() - - argdict = {} - - # do we need to skip 'opt' here? - # if line.startswith("opt "): - # line = line[4:] - - # get the bridge ID ? - if line.startswith("extra-info "): #XXX: get the router ID - line = line[11:] - (nickname, ID) = line.split() - logging.debug(" Parsed Nickname: %s", nickname) - if isValidFingerprint(ID): - logging.debug(" Parsed fingerprint: %s", ID) - ID = fromHex(ID) - else: - logging.debug(" Parsed invalid fingerprint: %s", ID) - - # get the transport line - if ID and line.startswith("transport "): - fields = line[10:].split() - - if len(fields) >= 3: - argdict = {} - # PT argumentss are comma-separated in the extrainfo - # descriptors. While there *shouldn't* be anything after them - # that was separated by a space (and hence would wind up being - # in a different `field`), if there was we'll join it to the - # rest of the PT arguments with a comma so that they are - # parsed as if they were PT arguments as well: - allargs = ','.join(fields[2:]) - for arg in allargs.split(','): - try: - k, v = arg.split('=') - except ValueError: - logging.warn(" Couldn't parse K=V from PT arg: %r" % arg) - else: - logging.debug(" Parsed PT Argument: %s: %s" % (k, v)) - argdict[k] = v - - # get the required fields, method name and address - if len(fields) >= 2: - # get the method name - # Method names must be C identifiers - for regex in [re_ipv4, re_ipv6]: - try: - method_name = re.match('[_a-zA-Z][_a-zA-Z0-9]*',fields[0]).group() - m = regex.match(fields[1]) - address = ipaddr.IPAddress(m.group(1)) - port = int(m.group(2)) - logging.debug(" Parsed Transport: %s %s:%d" - % (method_name, address, port)) - yield ID, method_name, address, port, argdict - except (IndexError, ValueError, AttributeError): - # skip this line - continue - - # end of descriptor is defined how? - if ID and line.startswith("router-signature"): - ID = None - -(a)deprecate.deprecated(Version('bridgedb', 0, 2, 4), - replacement='bridgedb.parse.descriptors.parseNetworkStatusFile') -def parseStatusFile(networkstatusFile): - """Parse entries in a bridge networkstatus file. - - :type networkstatusFile: A file-like object. - :param networkstatusFile: A file containing `@type bridge-networkstatus` documents. - """ - (nickname, ID, descDigest, timestamp, - ORaddr, ORport, dirport, addr, portlist) = (None for x in xrange(9)) - running = stable = False - parsedORAddressLines = 0 - or_addresses = {} - - for line in networkstatusFile: - line = line.strip() - if line.startswith("opt "): - line = line[4:] - - if line.startswith("r "): - (nickname, ID, descDigest, timestamp, - ORaddr, ORport, dirport) = networkstatus.parseRLine(line) - hexID = toHex(ID) - logging.debug("Parsed networkstatus line:") - logging.debug(" Nickname: %s" % nickname) - logging.debug(" Identity: %s" % hexID) - if descDigest: - descDigest = toHex(descDigest) - logging.debug(" Descriptor: {0}".format(descDigest)) - logging.debug(" Timestamp: {0}".format(timestamp)) - logging.debug(" ORAddress: {0}".format(ORaddr)) - logging.debug(" ORport: {0}".format(ORport)) - logging.debug(" dirport: {0}".format(dirport)) - - elif ID and line.startswith("a "): - try: - addr, portlist = networkstatus.parseALine(line, toHex(ID)) - except networkstatus.NetworkstatusParsingError as error: - logging.error(error) - else: - if (addr is not None) and (portlist is not None): - try: - or_addresses[addr].add(portlist) - except (KeyError, AttributeError): - or_addresses[addr] = portlist - parsedORAddressLines += 1 - - elif ID and timestamp and line.startswith("s "): - running, stable = networkstatus.parseSLine(line) - logging.debug("Bridges.parseStatusFile(): " - "yielding %s nickname=%s descDigest=%s " - "running=%s stable=%s oraddr=%s orport=%s " - "oraddrs=%s ts=%s" - % (hexID, nickname, descDigest, running, - stable, ORaddr, ORport, or_addresses, - timestamp)) - yield (ID, nickname, descDigest, running, stable, - ipaddr.IPAddress(ORaddr), ORport, - or_addresses, timestamp) - - (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport, - addr, portlist, hexID) = (None for x in xrange(10)) - running = stable = False - or_addresses = {} - - logging.debug("Total ORAddress lines parsed from '%s': %d" - % (networkstatusFile.name, parsedORAddressLines)) - - -(a)deprecate.deprecated( - Version('bridgedb', 0, 0, 1), - replacement='bridgedb.parse.networkstatus.NetworkstatusParsingError') -class ParseORAddressError(Exception): - def __init__(self, line=None): - msg = "Invalid or-address line" - if line: - msg += ": {0}".format(line) - Exception.__init__(self, msg) - - @deprecate.deprecated( Version('bridgedb', 0, 2, 4), replacement='bridgedb.bridges.Bridge') diff --git a/lib/bridgedb/test/deprecated_networkstatus.py b/lib/bridgedb/test/deprecated_networkstatus.py deleted file mode 100644 index 41d8c5f..0000000 --- a/lib/bridgedb/test/deprecated_networkstatus.py +++ /dev/null @@ -1,275 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of BridgeDB, a Tor bridge distribution system. -# -# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> -# please also see AUTHORS file -# :copyright: (c) 2013-2015 Isis Lovecruft -# (c) 2007-2015, The Tor Project, Inc. -# (c) 2007-2015, all entities within the AUTHORS file -# :license: 3-clause BSD, see included LICENSE for information - -"""Parsers for bridge networkstatus descriptors. - -THIS ENTIRE MODULE WAS DEPRECATED (AS PART OF #9380_) AND WAS REPLACED WITH -THE CORRESPONDING FUNCTIONS IN :mod:`bridgedb.parse.descriptors`. - -.. #9380: https://bugs.torproject.org/9380 - -.. py:module:: bridgedb.parse.networkstatus - :synopsis: Parsers for ``@type bridge-network-status`` descriptors_. -.. _descriptors: https://metrics.torproject.org/formats.html#descriptortypes - - -bridgedb.parse.networkstatus -============================ -:: - - networkstatus - |_ isValidRouterNickname - Determine if a nickname is according to spec - |_ parseRLine - Parse an 'r'-line from a networkstatus document - |_ parseALine - Parse an 'a'-line from a networkstatus document - \_ parseSLine - Parse an 's'-line from a networkstatus document -.. -""" - -import binascii -import logging -import string -import time -import warnings - -from twisted.python import deprecate -from twisted.python.log import showwarning -from twisted.python.versions import Version - -from bridgedb.parse import addr -from bridgedb.parse import parseUnpaddedBase64 -from bridgedb.parse import InvalidBase64 -from bridgedb.parse.nickname import InvalidRouterNickname -from bridgedb.parse.nickname import isValidRouterNickname - - -(a)deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -class NetworkstatusParsingError(Exception): - """Unable to parse networkstatus document line.""" - -(a)deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -class InvalidNetworkstatusRouterIdentity(ValueError): - """The ID field of a networkstatus document 'r'-line is invalid.""" - - -(a)deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -def parseRLine(line): - """Parse an 'r'-line from a networkstatus document. - - From torspec.git/dir-spec.txt, commit 36761c7d553d L1499-1512: - | "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort - | SP DirPort NL - | - | [At start, exactly once.] - | - | "Nickname" is the OR's nickname. "Identity" is a hash of its - | identity key, encoded in base64, with trailing equals sign(s) - | removed. "Digest" is a hash of its most recent descriptor as - | signed (that is, not including the signature), encoded in base64. - | "Publication" is the - | publication time of its most recent descriptor, in the form - | YYYY-MM-DD HH:MM:SS, in UTC. "IP" is its current IP address; - | ORPort is its current OR port, "DirPort" is its current directory - | port, or "0" for "none". - - :param string line: An 'r'-line from an bridge-network-status descriptor. - :returns: - A 7-tuple of:: - (nickname, identityDigest, descriptorDigest, timestamp, - orAddress, orPort, dirport) - where each value is set according to the data parsed from the - **line**, or ``None`` if nothing suitable could be parsed. - """ - (nickname, ID, descDigest, timestamp, - ORaddr, ORport, dirport) = (None for x in xrange(7)) - - try: - if not line.startswith('r '): - raise NetworkstatusParsingError( - "Networkstatus parser received non 'r'-line: %r" % line) - - line = line[2:] # Chop off the 'r ' - fields = line.split() - - if len(fields) != 8: - raise NetworkstatusParsingError( - "Wrong number of fields in networkstatus 'r'-line: %r" % line) - - nickname, ID = fields[:2] - - try: - ID = parseUnpaddedBase64(ID) - except InvalidBase64 as error: - raise InvalidNetworkstatusRouterIdentity(error) - - # Check the nickname validity after parsing the ID, otherwise, if the - # nickname is invalid, we end up with the nickname being ``None`` and - # the ID being unparsed, unpadded (meaning it is technically invalid) - # base64. - isValidRouterNickname(nickname) - - except NetworkstatusParsingError as error: - logging.error(error) - nickname, ID = None, None - except InvalidRouterNickname as error: - logging.error(error) - # Assume that we mostly care about obtaining the OR's ID, then it - # should be okay to set the nickname to ``None``, if it was invalid. - nickname = None - except InvalidNetworkstatusRouterIdentity as error: - logging.error(error) - ID = None - else: - try: - descDigest = parseUnpaddedBase64(fields[2]) - timestamp = time.mktime(time.strptime(" ".join(fields[3:5]), - "%Y-%m-%d %H:%M:%S")) - ORaddr = fields[5] - ORport = int(fields[6]) - dirport = fields[7] - except InvalidBase64 as error: - logging.error(error) - descDigest = None - except (AttributeError, ValueError, IndexError) as error: - logging.error(error) - timestamp = None - finally: - return (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport) - -(a)deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -def parseALine(line, fingerprint=None): - """Parse an 'a'-line of a bridge networkstatus document. - - From torspec.git/dir-spec.txt, commit 36761c7d553d L1499-1512: - | - | "a" SP address ":" port NL - | - | [Any number.] - | - | Present only if the OR has at least one IPv6 address. - | - | Address and portlist are as for "or-address" as specified in - | 2.1. - | - | (Only included when the vote or consensus is generated with - | consensus-method 14 or later.) - - :param string line: An 'a'-line from an bridge-network-status descriptor. - :type fingerprint: string or None - :param fingerprint: A string which identifies which OR the descriptor - we're parsing came from (since the 'a'-line doesn't tell us, this can - help make the log messages clearer). - :raises: :exc:`NetworkstatusParsingError` - :rtype: tuple - :returns: A 2-tuple of a string respresenting the IP address and a - :class:`bridgedb.parse.addr.PortList`. - """ - ip = None - portlist = None - - if line.startswith('a '): - line = line[2:] # Chop off the 'a ' - else: - logging.warn("Networkstatus parser received non 'a'-line for %r:"\ - " %r" % (fingerprint or 'Unknown', line)) - - try: - ip, portlist = line.rsplit(':', 1) - except ValueError as error: - logging.error("Bad separator in networkstatus 'a'-line: %r" % line) - return (None, None) - - if ip.startswith('[') and ip.endswith(']'): - ip = ip.strip('[]') - - try: - if not addr.isIPAddress(ip): - raise NetworkstatusParsingError( - "Got invalid IP Address in networkstatus 'a'-line for %r: %r" - % (fingerprint or 'Unknown', line)) - - if addr.isIPv4(ip): - warnings.warn(FutureWarning( - "Got IPv4 address in networkstatus 'a'-line! "\ - "Networkstatus document format may have changed!")) - except NetworkstatusParsingError as error: - logging.error(error) - ip, portlist = None, None - - try: - portlist = addr.PortList(portlist) - if not portlist: - raise NetworkstatusParsingError( - "Got invalid portlist in 'a'-line for %r!\n Line: %r" - % (fingerprint or 'Unknown', line)) - except (addr.InvalidPort, NetworkstatusParsingError) as error: - logging.error(error) - portlist = None - else: - logging.debug("Parsed networkstatus ORAddress line for %r:"\ - "\n Address: %s \tPorts: %s" - % (fingerprint or 'Unknown', ip, portlist)) - finally: - return (ip, portlist) - -(a)deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -def parseSLine(line): - """Parse an 's'-line from a bridge networkstatus document. - - The 's'-line contains all flags assigned to a bridge. The flags which may - be assigned to a bridge are as follows: - - From torspec.git/dir-spec.txt, commit 36761c7d553d L1526-1554: - | - | "s" SP Flags NL - | - | [Exactly once.] - | - | A series of space-separated status flags, in lexical order (as ASCII - | byte strings). Currently documented flags are: - | - | "BadDirectory" if the router is believed to be useless as a - | directory cache (because its directory port isn't working, - | its bandwidth is always throttled, or for some similar - | reason). - | "Fast" if the router is suitable for high-bandwidth circuits. - | "Guard" if the router is suitable for use as an entry guard. - | "HSDir" if the router is considered a v2 hidden service directory. - | "Named" if the router's identity-nickname mapping is canonical, - | and this authority binds names. - | "Stable" if the router is suitable for long-lived circuits. - | "Running" if the router is currently usable. - | "Valid" if the router has been 'validated'. - | "V2Dir" if the router implements the v2 directory protocol. - - :param string line: An 's'-line from an bridge-network-status descriptor. - :rtype: tuple - :returns: A 2-tuple of booleans, the first is True if the bridge has the - "Running" flag, and the second is True if it has the "Stable" flag. - """ - line = line[2:] - - flags = [x.capitalize() for x in line.split()] - fast = 'Fast' in flags - running = 'Running' in flags - stable = 'Stable' in flags - guard = 'Guard' in flags - valid = 'Valid' in flags - - if (fast or running or stable or guard or valid): - logging.debug("Parsed Flags: %s%s%s%s%s" - % ('Fast ' if fast else '', - 'Running ' if running else '', - 'Stable ' if stable else '', - 'Guard ' if guard else '', - 'Valid ' if valid else '')) - - # Right now, we only care about 'Running' and 'Stable' - return running, stable diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index a039f91..571b3ca 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -34,7 +34,6 @@ from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.Stability import BridgeHistory from bridgedb.parse import addr -from bridgedb.test import deprecated_networkstatus as networkstatus from bridgedb.test.util import bracketIPv6 from bridgedb.test.util import randomIP from bridgedb.test.util import randomIPv4 diff --git a/lib/bridgedb/test/test_Tests.py b/lib/bridgedb/test/test_Tests.py index e78a4c4..84c3f8b 100644 --- a/lib/bridgedb/test/test_Tests.py +++ b/lib/bridgedb/test/test_Tests.py @@ -84,17 +84,6 @@ def monkeypatchTests(): patches from :mod:`bridgedb.test.deprecated`. """ patcher = monkey.MonkeyPatcher() - patcher.addPatch(Tests.networkstatus, 'parseALine', - deprecated.parseORAddressLine) - patcher.addPatch(Tests.addr, 'PortList', deprecated.PortList) - patcher.addPatch(Tests.bridgedb.Bridges, 'PORTSPEC_LEN', 16) - patcher.addPatch(Tests.bridgedb.Bridges, 're_ipv4', deprecated.re_ipv4) - patcher.addPatch(Tests.bridgedb.Bridges, 're_ipv6', deprecated.re_ipv6) - patcher.addPatch(Tests.bridgedb.Bridges, 'HEX_FP_LEN', 40) - patcher.addPatch(Tests.bridgedb.Bridges, 'toHex', binascii.b2a_hex) - patcher.addPatch(Tests.bridgedb.Bridges, 'fromHex', binascii.a2b_hex) - patcher.addPatch(Tests.bridgedb.Bridges, 'is_valid_fingerprint', - deprecated.is_valid_fingerprint) patcher.addPatch(Tests.bridgedb.Bridges, 'PluggableTransport', deprecated.PluggableTransport) patcher.addPatch(Tests.bridgedb.Bridges, 'Bridge',

1 0

[bridgedb/develop] Remove unused contextlib import in bridgedb.Storage.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit 913507e585de16e416f017050500262d2f1aa56a Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 11 20:20:46 2015 +0000 Remove unused contextlib import in bridgedb.Storage. --- lib/bridgedb/Storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index f9fcf43..ef76829 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -9,7 +9,6 @@ import binascii import sqlite3 import time import hashlib -from contextlib import contextmanager from contextlib import GeneratorContextManager from functools import wraps from ipaddr import IPAddress, IPv6Address, IPv4Address

1 0

[bridgedb/develop] Move Main.updateBridgeHistory() → Stability.updateBridgeHistory().
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit 6417ffe9681138ac1fb9dbccd897df61ef358ae1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 10 23:59:42 2015 +0000 Move Main.updateBridgeHistory() → Stability.updateBridgeHistory(). Conflicts: lib/bridgedb/Main.py --- lib/bridgedb/Main.py | 42 ++-------------------------------------- lib/bridgedb/Stability.py | 40 ++++++++++++++++++++++++++++++++++++++ lib/bridgedb/test/test_Main.py | 28 ++++++++++++++------------- 3 files changed, 57 insertions(+), 53 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 88b386b..4a00799 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -40,51 +40,13 @@ from bridgedb.configure import Conf from bridgedb.parse import descriptors from bridgedb.parse import options from bridgedb.parse.addr import isIPAddress -from bridgedb.schedule import toUnixSeconds import bridgedb.Storage from bridgedb import Bridges from bridgedb import Dist -from bridgedb.Stability import addOrUpdateBridgeHistory - - -def updateBridgeHistory(bridges, timestamps): - """Process all the timestamps and update the bridge stability statistics in - the database. - - .. warning: This function is extremely expensive, and will keep getting - more and more expensive, on a linearithmic scale, every time it is - called. Blame the :mod:`bridgedb.Stability` module. - - :param dict bridges: All bridges from the descriptors, parsed into - :class:`bridgedb.bridges.Bridge`s. - :param dict timestamps: A dictionary whose keys are bridge fingerprints, - and whose values are lists of integers, each integer being a timestamp - (in seconds since Unix Epoch) for when a descriptor for that bridge - was published. - :rtype: dict - :returns: The original **timestamps**, but which each list of integers - (re)sorted. - """ - logging.debug("Beginning bridge stability calculations") - sortedTimestamps = {} - - for fingerprint, stamps in timestamps.items()[:]: - stamps.sort() - bridge = bridges[fingerprint] - for timestamp in stamps: - logging.debug( - ("Adding/updating timestamps in BridgeHistory for %s in " - "database: %s") % (fingerprint, timestamp)) - timestamp = toUnixSeconds(timestamp.timetuple()) - addOrUpdateBridgeHistory(bridge, timestamp) - # Replace the timestamps so the next sort is (hopefully) less - # expensive: - sortedTimestamps[fingerprint] = stamps - - logging.debug("Stability calculations complete") - return sortedTimestamps +from bridgedb.Stability import updateBridgeHistory + def load(state, splitter, clear=False): """Read and parse all descriptors, and load into a bridge splitter. diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index ad6fd0c..fbc8e88 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -28,6 +28,9 @@ Bridge stability metrics are calculated using the model introduced in import logging import bridgedb.Storage +from bridgedb.schedule import toUnixSeconds + + # tunables weighting_factor = float(19)/float(20) discountIntervalMillis = long(60*60*12*1000) @@ -253,3 +256,40 @@ def updateWeightedTime(statusPublicationMillis): for bh in bhToUpdate: db.updateIntoBridgeHistory(bh) + +def updateBridgeHistory(bridges, timestamps): + """Process all the timestamps and update the bridge stability statistics in + the database. + + .. warning: This function is extremely expensive, and will keep getting + more and more expensive, on a linearithmic scale, every time it is + called. Blame the :mod:`bridgedb.Stability` module. + + :param dict bridges: All bridges from the descriptors, parsed into + :class:`bridgedb.bridges.Bridge`s. + :param dict timestamps: A dictionary whose keys are bridge fingerprints, + and whose values are lists of integers, each integer being a timestamp + (in seconds since Unix Epoch) for when a descriptor for that bridge + was published. + :rtype: dict + :returns: The original **timestamps**, but which each list of integers + (re)sorted. + """ + logging.debug("Beginning bridge stability calculations") + sortedTimestamps = {} + + for fingerprint, stamps in timestamps.items()[:]: + stamps.sort() + bridge = bridges[fingerprint] + for timestamp in stamps: + logging.debug( + ("Adding/updating timestamps in BridgeHistory for %s in " + "database: %s") % (fingerprint, timestamp)) + timestamp = toUnixSeconds(timestamp.timetuple()) + addOrUpdateBridgeHistory(bridge, timestamp) + # Replace the timestamps so the next sort is (hopefully) less + # expensive: + sortedTimestamps[fingerprint] = stamps + + logging.debug("Stability calculations complete") + return sortedTimestamps diff --git a/lib/bridgedb/test/test_Main.py b/lib/bridgedb/test/test_Main.py index 0b14319..5ae4025 100644 --- a/lib/bridgedb/test/test_Main.py +++ b/lib/bridgedb/test/test_Main.py @@ -48,13 +48,15 @@ w Bandwidth=2094050 p reject 1-65535 ''' -def mockAddOrUpdateBridgeHistory(bridge, timestamp): - """A mocked version of :func:`bridgedb.Stability.addOrUpdateBridgeHistory` +def mockUpdateBridgeHistory(bridges, timestamps): + """A mocked version of :func:`bridgedb.Stability.updateBridgeHistory` which doesn't access the database (so that we can test functions which - call it, like :func:`bridgedb.Main.updateBridgeHistory`). + call it, like :func:`bridgedb.Main.load`). """ - print("Pretending to update Bridge %s with timestamp %s..." % - (bridge.fingerprint, timestamp)) + for fingerprint, stamps in timestamps.items()[:]: + for timestamp in stamps: + print("Pretending to update Bridge %s with timestamp %s..." % + (fingerprint, timestamp)) class MockBridgeHolder(BridgeHolder): @@ -170,23 +172,23 @@ class MainTests(unittest.TestCase): # Functions which some tests mock, which we'll need to re-replace # later in tearDown(): - self._orig_addOrUpdateBridgeHistory = Main.addOrUpdateBridgeHistory + self._orig_updateBridgeHistory = Main.updateBridgeHistory self._orig_sys_argv = sys.argv def tearDown(self): - """Replace the mocked mockAddOrUpdateBridgeHistory() function with the - real function, Stability.addOrUpdateBridgeHistory(). + """Replace the mocked mockUpdateBridgeHistory() function with the + real function, Stability.updateBridgeHistory(). """ - Main.addOrUpdateBridgeHistory = self._orig_addOrUpdateBridgeHistory + Main.updateBridgeHistory = self._orig_updateBridgeHistory sys.argv = self._orig_sys_argv def test_Main_updateBridgeHistory(self): """Main.updateBridgeHistory should update some timestamps for some bridges. """ - # Mock the addOrUpdateBridgeHistory() function so that we don't try to + # Mock the updateBridgeHistory() function so that we don't try to # access the database: - Main.addOrUpdateBridgeHistory = mockAddOrUpdateBridgeHistory + Main.updateBridgeHistory = mockUpdateBridgeHistory # Get the bridges into the mocked splitter d = deferToThread(Main.load, self.state, self.splitter) @@ -222,12 +224,12 @@ class MainTests(unittest.TestCase): """ # Mock the addOrUpdateBridgeHistory() function so that we don't try to # access the database: - Main.addOrUpdateBridgeHistory = mockAddOrUpdateBridgeHistory + Main.updateBridgeHistory = mockUpdateBridgeHistory state = self.state state.COLLECT_TIMESTAMPS = True # The reactor is deferring this to a thread, so the test execution - # here isn't actually covering the Main.updateBridgeHistory() + # here isn't actually covering the Storage.updateBridgeHistory() # function: Main.load(state, self.splitter)

1 0

[bridgedb/develop] Remove remnants of `bridgedb test` options handling.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit d90e75f7b55ecead452255db1074f8fe7b39d04a Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 11 00:10:30 2015 +0000 Remove remnants of `bridgedb test` options handling. --- lib/bridgedb/Main.py | 6 ------ lib/bridgedb/parse/options.py | 1 - 2 files changed, 7 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 4a00799..d6b31a9 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -524,12 +524,6 @@ def runSubcommand(options, config): statuscode = runner.generateDescriptors( options.subOptions['descriptors'], config.RUN_IN_DIR) - if options.subCommand == 'test': - if options.subOptions['trial']: - runner.runTrial(options.subOptions) - if options.subOptions['unittests']: - runner.runTests(options.subOptions) - logging.info("Subcommand '%s' finished with status %s." % (options.subCommand, statuscode)) sys.exit(statuscode) diff --git a/lib/bridgedb/parse/options.py b/lib/bridgedb/parse/options.py index 3979615..6f3bfc1 100644 --- a/lib/bridgedb/parse/options.py +++ b/lib/bridgedb/parse/options.py @@ -36,7 +36,6 @@ bridgedb.parse.options | |__ MockOptions - Suboptions for creating fake bridge descriptors for | testing purposes. - |__ TestOptions - Suboptions for running twisted.trial based unittests. \__ MainOptions - Main commandline options parser for BridgeDB. .. """

1 0

[bridgedb/develop] Fix a unittest for PluggableTransport.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit 31a6b5a18ba9b5319d1c32a1fda147f502b52827 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 10 22:57:28 2015 +0000 Fix a unittest for PluggableTransport. The ptargs was supposed to be a list, and previously it was erroneously a string. --- lib/bridgedb/test/test_bridges.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index 99fc458..88b858e 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -1715,7 +1715,8 @@ class BridgeTests(unittest.TestCase): self.bridge.updateFromServerDescriptor(self.serverdescriptor) obfs4 = self.extrainfo.transport['obfs4'] - obfs4 = (u'1.1.1.1', 1111, obfs4[-1][-1].replace('iat-mode=0,', '')) + ptargs = [str(obfs4[-1][-1].replace('iat-mode=0', '')),] + obfs4 = (u'1.1.1.1', 1111, ptargs) self.extrainfo.transport['obfs4'] = obfs4 self.bridge.updateFromExtraInfoDescriptor(self.extrainfo)

1 0

[bridgedb/develop] Cleanup bridgedb.Stability module docstring and add file header.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit 62701b3c3673ad474831363827bcb2a22580bef6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 10 23:13:42 2015 +0000 Cleanup bridgedb.Stability module docstring and add file header. --- lib/bridgedb/Stability.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index 981e005..ad6fd0c 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -1,13 +1,30 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_Stability -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: please see the AUTHORS file for attributions +# :copyright: (c) 2013-2015, Isis Lovecruft +# (c) 2013-2015, Matthew Finkel +# (c) 2012-2015, Aaron Gibson +# (c) 2007-2015, Nick Mathewson +# (c) 2007-2015, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +"""This module provides functionality for tracking bridge stability metrics. + +Bridge stability metrics are calculated using the model introduced in +`"An Analysis of Tor Bridge Stability"`_ and +`implemented in the Tor Metrics library`_. + +.. An Analysis of Tor Bridge Stability: + https://metrics.torproject.org/papers/bridge-stability-2011-10-31.pdf + Karsten Loesing, An Analysis of Tor Bridge Stability. Technical Report. + The Tor Project, October 2011. + +.. implemented in the Tor Metrics library: + https://gitweb.torproject.org/metrics-tasks/task-4255/SimulateBridgeStabili… """ -This module provides functionality for tracking bridge stability metrics, using -the model introduced in [1] and implemented in [2]. -[1] Karsten Loesing, An Analysis of Tor Bridge Stability. Technical Report. -The Tor Project, October 2011. -https://metrics.torproject.org/papers/bridge-stability-2011-10-31.pdf - -[2] https://gitweb.torproject.org/metrics-tasks/task-4255/SimulateBridgeStabili… -""" import logging import bridgedb.Storage @@ -15,6 +32,7 @@ import bridgedb.Storage weighting_factor = float(19)/float(20) discountIntervalMillis = long(60*60*12*1000) + class BridgeHistory(object): """ Record Class that tracks a single Bridge The fields stored are:

1 0

[bridgedb/develop] Decrease logging for extrainfo descriptor deduplication.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit ccb709c56eb42dfb2a3fb10540c7d2d303c78c68 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 11 20:19:32 2015 +0000 Decrease logging for extrainfo descriptor deduplication. It was *really* noisy. --- lib/bridgedb/parse/descriptors.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/bridgedb/parse/descriptors.py b/lib/bridgedb/parse/descriptors.py index 3f8d48e..b90e2e7 100644 --- a/lib/bridgedb/parse/descriptors.py +++ b/lib/bridgedb/parse/descriptors.py @@ -189,9 +189,6 @@ def deduplicate(descriptors, statistics=False): for descriptor in descriptors: fingerprint = descriptor.fingerprint - logging.debug("Deduplicating %s descriptor for router %s" - % (descriptor.__class__.__name__.rsplit('.', 1)[-1], - safelog.logSafely(fingerprint))) if fingerprint in duplicates: duplicates[fingerprint].append(descriptor) else:

1 0

[bridgedb/develop] Fix idempotency and reentrancy issues with Bridge.allVanillaAddresses.
by isis＠torproject.org 02 Jun '15

02 Jun '15

commit 6e60cc33bac91442ed7b45994ef312ebd86916bd Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 10 22:38:38 2015 +0000 Fix idempotency and reentrancy issues with Bridge.allVanillaAddresses. Conflicts: lib/bridgedb/bridges.py --- lib/bridgedb/bridges.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 37e17cd..e1c8738 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -1105,10 +1105,23 @@ class Bridge(BridgeBackwardsCompatibility): :returns: All of this bridge's ORAddresses, as well as its ORPort IP address and port. """ - addresses = self.orAddresses - # Add the default ORPort address (it will always be IPv4, otherwise - # Stem should have raised a ValueError during parsing): - addresses.append((self.address, self.orPort, 4)) + # Force deep-copying of the orAddresses. Otherwise, the later use of + # ``addresses.append()`` is both non-reentrant and non-idempotent, as + # it would change the value of ``Bridge.orAddresses``, as well as + # append a (possibly updated, if ``Bridge.address`` or + # ``Bridge.orPort`` changed!) new copy of the bridge's primary + # ORAddress each time this property is called. + addresses = self.orAddresses[:] + + # Add the default ORPort address. It will always be IPv4, otherwise + # Stem should have raised a ValueError during parsing. But for + # testability, check which type it is: + version = 4 + if isIPv6(self.address): + version = 6 + + addresses.append((self.address, self.orPort, version)) + return addresses def assertOK(self):

1 0