commit 452f2aa9c5cfb4f6c80615c89de75f9f45c0d685 Author: Isis Lovecruft isis@torproject.org Date: Mon May 11 22:42:06 2015 +0000
Remove integration test monkeypatching for very old, deprecated parsers.
* REMOVE bridgedb.test.deprecated_networkstatus. * REMOVE descriptor parsers from bridgedb.test.deprecated (which were deprecated by bridgedb.test.deprecated_networkstatus, which was in turn deprecated by the Stem-based parsers in bridgedb.parse.descriptors as part of #9380). * REMOVE monkeypatching of old unittests with (doubly) deprecated parsers.
Conflicts: lib/bridgedb/test/deprecated.py lib/bridgedb/test/test_Tests.py --- lib/bridgedb/test/deprecated.py | 254 ----------------------- lib/bridgedb/test/deprecated_networkstatus.py | 275 ------------------------- lib/bridgedb/test/legacy_Tests.py | 1 - lib/bridgedb/test/test_Tests.py | 11 - 4 files changed, 541 deletions(-)
diff --git a/lib/bridgedb/test/deprecated.py b/lib/bridgedb/test/deprecated.py index 181814c..df660b6 100644 --- a/lib/bridgedb/test/deprecated.py +++ b/lib/bridgedb/test/deprecated.py @@ -20,260 +20,6 @@ from twisted.python import deprecate from twisted.python.versions import Version
-PORTSPEC_LENGTH = 16 - -re_ipv6 = re.compile("[([a-fA-F0-9:]+)]:(.*$)") -re_ipv4 = re.compile("((?:\d{1,3}.?){4}):(.*$)") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 0, 1), - "Removed due to 'bridgedb.Bridges.PortList' being moved to "\ - "'bridgedb.parse.addr.PortList.", - "bridgedb.Bridges", - "PORTSPEC_LENGTH") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 0, 1), - "Attribute 'bridgedb.Bridges.re_ipv4' was removed due to "\ - "'bridgedb.Bridges.parseORAddressLine' moving to "\ - "'bridgedb.parse.networkstatus.", - "bridgedb.Bridges", - "re_ipv4") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 0, 1), - "Attribute 'bridgedb.Bridges.re_ipv6' was removed due to "\ - "'bridgedb.Bridges.parseORAddressLine' moving to "\ - "'bridgedb.parse.networkstatus.", - "bridgedb.Bridges", - "re_ipv6") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 2, 3), - ("Removed due to 'bridgedb.Bridges.HEX_FP_LEN' being moved to " - "'bridgedb.parse.fingerprint.HEX_FINGERPRINT_LEN."), - "bridgedb.Bridges", "HEX_FP_LEN") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 2, 3), - ("Removed due to 'bridgedb.Bridges.toHex' being moved to " - "'bridgedb.parse.fingerprint.toHex."), - "bridgedb.Bridges", "toHex") - -deprecate.deprecatedModuleAttribute( - Version('bridgedb', 0, 2, 3), - ("Removed due to 'bridgedb.Bridges.fromHex' being moved to " - "'bridgedb.parse.fingerprint.fromHex."), - "bridgedb.Bridges", "fromHex") - - -@deprecate.deprecated( - Version('bridgedb', 0, 0, 1), - replacement='bridgedb.parse.networkstatus.parseALine') -def parseORAddressLine(line): - """Deprecated :func:`bridgedb.Bridges.parseORAddressLine`, removed in - bridgedb-0.1.0, in commit 1f111e5. - - This function and the newer parsers from :mod:`bridgedb.parse.netstatus` - are alternately :api:`~twisted.python.monkey.MonkeyPatcher.patch`ed into - the :mod:`old unittests <bridgedb.Tests>`, so that the later functions as - a suite of regression tests. - """ - address = None - portlist = None - # try regexp to discover ip version - for regex in [re_ipv4, re_ipv6]: - m = regex.match(line) - if m: - # get an address and portspec, or raise ParseError - try: - address = ipaddr.IPAddress(m.group(1)) - portlist = PortList(m.group(2)) - except (IndexError, ValueError): raise ParseORAddressError(line) - - # return a valid address, portlist or raise ParseORAddressError - if address and portlist and len(portlist): return address,portlist - raise ParseORAddressError(line) - -@deprecate.deprecated(Version('bridgedb', 0, 2, 3), - replacement='bridgedb.parse.fingerprint.isValidFingerprint') -def is_valid_fingerprint(fp): - """Return true iff fp in the right format to be a hex fingerprint - of a Tor server. - """ - if len(fp) != HEX_FP_LEN: - return False - try: - fromHex(fp) - except TypeError: - return False - else: - return True - -@deprecate.deprecated(Version('bridgedb', 0, 2, 4), - replacement='bridgedb.parse.descriptors.parseExtraInfoFiles') -def parseExtraInfoFile(f): - """ - parses lines in Bridges extra-info documents. - returns an object whose type corresponds to the - relevant set of extra-info lines. - - presently supported lines and the accompanying type are: - - { 'transport': PluggableTransport, } - - 'transport' lines (torspec.git/proposals/180-pluggable-transport.txt) - - Bridges put the 'transport' lines in their extra-info documents. - the format is: - - transport SP <methodname> SP address:port [SP arglist] NL - """ - - ID = None - for line in f: - line = line.strip() - - argdict = {} - - # do we need to skip 'opt' here? - # if line.startswith("opt "): - # line = line[4:] - - # get the bridge ID ? - if line.startswith("extra-info "): #XXX: get the router ID - line = line[11:] - (nickname, ID) = line.split() - logging.debug(" Parsed Nickname: %s", nickname) - if isValidFingerprint(ID): - logging.debug(" Parsed fingerprint: %s", ID) - ID = fromHex(ID) - else: - logging.debug(" Parsed invalid fingerprint: %s", ID) - - # get the transport line - if ID and line.startswith("transport "): - fields = line[10:].split() - - if len(fields) >= 3: - argdict = {} - # PT argumentss are comma-separated in the extrainfo - # descriptors. While there *shouldn't* be anything after them - # that was separated by a space (and hence would wind up being - # in a different `field`), if there was we'll join it to the - # rest of the PT arguments with a comma so that they are - # parsed as if they were PT arguments as well: - allargs = ','.join(fields[2:]) - for arg in allargs.split(','): - try: - k, v = arg.split('=') - except ValueError: - logging.warn(" Couldn't parse K=V from PT arg: %r" % arg) - else: - logging.debug(" Parsed PT Argument: %s: %s" % (k, v)) - argdict[k] = v - - # get the required fields, method name and address - if len(fields) >= 2: - # get the method name - # Method names must be C identifiers - for regex in [re_ipv4, re_ipv6]: - try: - method_name = re.match('[_a-zA-Z][_a-zA-Z0-9]*',fields[0]).group() - m = regex.match(fields[1]) - address = ipaddr.IPAddress(m.group(1)) - port = int(m.group(2)) - logging.debug(" Parsed Transport: %s %s:%d" - % (method_name, address, port)) - yield ID, method_name, address, port, argdict - except (IndexError, ValueError, AttributeError): - # skip this line - continue - - # end of descriptor is defined how? - if ID and line.startswith("router-signature"): - ID = None - -@deprecate.deprecated(Version('bridgedb', 0, 2, 4), - replacement='bridgedb.parse.descriptors.parseNetworkStatusFile') -def parseStatusFile(networkstatusFile): - """Parse entries in a bridge networkstatus file. - - :type networkstatusFile: A file-like object. - :param networkstatusFile: A file containing `@type bridge-networkstatus` documents. - """ - (nickname, ID, descDigest, timestamp, - ORaddr, ORport, dirport, addr, portlist) = (None for x in xrange(9)) - running = stable = False - parsedORAddressLines = 0 - or_addresses = {} - - for line in networkstatusFile: - line = line.strip() - if line.startswith("opt "): - line = line[4:] - - if line.startswith("r "): - (nickname, ID, descDigest, timestamp, - ORaddr, ORport, dirport) = networkstatus.parseRLine(line) - hexID = toHex(ID) - logging.debug("Parsed networkstatus line:") - logging.debug(" Nickname: %s" % nickname) - logging.debug(" Identity: %s" % hexID) - if descDigest: - descDigest = toHex(descDigest) - logging.debug(" Descriptor: {0}".format(descDigest)) - logging.debug(" Timestamp: {0}".format(timestamp)) - logging.debug(" ORAddress: {0}".format(ORaddr)) - logging.debug(" ORport: {0}".format(ORport)) - logging.debug(" dirport: {0}".format(dirport)) - - elif ID and line.startswith("a "): - try: - addr, portlist = networkstatus.parseALine(line, toHex(ID)) - except networkstatus.NetworkstatusParsingError as error: - logging.error(error) - else: - if (addr is not None) and (portlist is not None): - try: - or_addresses[addr].add(portlist) - except (KeyError, AttributeError): - or_addresses[addr] = portlist - parsedORAddressLines += 1 - - elif ID and timestamp and line.startswith("s "): - running, stable = networkstatus.parseSLine(line) - logging.debug("Bridges.parseStatusFile(): " - "yielding %s nickname=%s descDigest=%s " - "running=%s stable=%s oraddr=%s orport=%s " - "oraddrs=%s ts=%s" - % (hexID, nickname, descDigest, running, - stable, ORaddr, ORport, or_addresses, - timestamp)) - yield (ID, nickname, descDigest, running, stable, - ipaddr.IPAddress(ORaddr), ORport, - or_addresses, timestamp) - - (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport, - addr, portlist, hexID) = (None for x in xrange(10)) - running = stable = False - or_addresses = {} - - logging.debug("Total ORAddress lines parsed from '%s': %d" - % (networkstatusFile.name, parsedORAddressLines)) - - -@deprecate.deprecated( - Version('bridgedb', 0, 0, 1), - replacement='bridgedb.parse.networkstatus.NetworkstatusParsingError') -class ParseORAddressError(Exception): - def __init__(self, line=None): - msg = "Invalid or-address line" - if line: - msg += ": {0}".format(line) - Exception.__init__(self, msg) - - @deprecate.deprecated( Version('bridgedb', 0, 2, 4), replacement='bridgedb.bridges.Bridge') diff --git a/lib/bridgedb/test/deprecated_networkstatus.py b/lib/bridgedb/test/deprecated_networkstatus.py deleted file mode 100644 index 41d8c5f..0000000 --- a/lib/bridgedb/test/deprecated_networkstatus.py +++ /dev/null @@ -1,275 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of BridgeDB, a Tor bridge distribution system. -# -# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org -# please also see AUTHORS file -# :copyright: (c) 2013-2015 Isis Lovecruft -# (c) 2007-2015, The Tor Project, Inc. -# (c) 2007-2015, all entities within the AUTHORS file -# :license: 3-clause BSD, see included LICENSE for information - -"""Parsers for bridge networkstatus descriptors. - -THIS ENTIRE MODULE WAS DEPRECATED (AS PART OF #9380_) AND WAS REPLACED WITH -THE CORRESPONDING FUNCTIONS IN :mod:`bridgedb.parse.descriptors`. - -.. #9380: https://bugs.torproject.org/9380 - -.. py:module:: bridgedb.parse.networkstatus - :synopsis: Parsers for ``@type bridge-network-status`` descriptors_. -.. _descriptors: https://metrics.torproject.org/formats.html#descriptortypes - - -bridgedb.parse.networkstatus -============================ -:: - - networkstatus - |_ isValidRouterNickname - Determine if a nickname is according to spec - |_ parseRLine - Parse an 'r'-line from a networkstatus document - |_ parseALine - Parse an 'a'-line from a networkstatus document - _ parseSLine - Parse an 's'-line from a networkstatus document -.. -""" - -import binascii -import logging -import string -import time -import warnings - -from twisted.python import deprecate -from twisted.python.log import showwarning -from twisted.python.versions import Version - -from bridgedb.parse import addr -from bridgedb.parse import parseUnpaddedBase64 -from bridgedb.parse import InvalidBase64 -from bridgedb.parse.nickname import InvalidRouterNickname -from bridgedb.parse.nickname import isValidRouterNickname - - -@deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -class NetworkstatusParsingError(Exception): - """Unable to parse networkstatus document line.""" - -@deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -class InvalidNetworkstatusRouterIdentity(ValueError): - """The ID field of a networkstatus document 'r'-line is invalid.""" - - -@deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -def parseRLine(line): - """Parse an 'r'-line from a networkstatus document. - - From torspec.git/dir-spec.txt, commit 36761c7d553d L1499-1512: - | "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort - | SP DirPort NL - | - | [At start, exactly once.] - | - | "Nickname" is the OR's nickname. "Identity" is a hash of its - | identity key, encoded in base64, with trailing equals sign(s) - | removed. "Digest" is a hash of its most recent descriptor as - | signed (that is, not including the signature), encoded in base64. - | "Publication" is the - | publication time of its most recent descriptor, in the form - | YYYY-MM-DD HH:MM:SS, in UTC. "IP" is its current IP address; - | ORPort is its current OR port, "DirPort" is its current directory - | port, or "0" for "none". - - :param string line: An 'r'-line from an bridge-network-status descriptor. - :returns: - A 7-tuple of:: - (nickname, identityDigest, descriptorDigest, timestamp, - orAddress, orPort, dirport) - where each value is set according to the data parsed from the - **line**, or ``None`` if nothing suitable could be parsed. - """ - (nickname, ID, descDigest, timestamp, - ORaddr, ORport, dirport) = (None for x in xrange(7)) - - try: - if not line.startswith('r '): - raise NetworkstatusParsingError( - "Networkstatus parser received non 'r'-line: %r" % line) - - line = line[2:] # Chop off the 'r ' - fields = line.split() - - if len(fields) != 8: - raise NetworkstatusParsingError( - "Wrong number of fields in networkstatus 'r'-line: %r" % line) - - nickname, ID = fields[:2] - - try: - ID = parseUnpaddedBase64(ID) - except InvalidBase64 as error: - raise InvalidNetworkstatusRouterIdentity(error) - - # Check the nickname validity after parsing the ID, otherwise, if the - # nickname is invalid, we end up with the nickname being ``None`` and - # the ID being unparsed, unpadded (meaning it is technically invalid) - # base64. - isValidRouterNickname(nickname) - - except NetworkstatusParsingError as error: - logging.error(error) - nickname, ID = None, None - except InvalidRouterNickname as error: - logging.error(error) - # Assume that we mostly care about obtaining the OR's ID, then it - # should be okay to set the nickname to ``None``, if it was invalid. - nickname = None - except InvalidNetworkstatusRouterIdentity as error: - logging.error(error) - ID = None - else: - try: - descDigest = parseUnpaddedBase64(fields[2]) - timestamp = time.mktime(time.strptime(" ".join(fields[3:5]), - "%Y-%m-%d %H:%M:%S")) - ORaddr = fields[5] - ORport = int(fields[6]) - dirport = fields[7] - except InvalidBase64 as error: - logging.error(error) - descDigest = None - except (AttributeError, ValueError, IndexError) as error: - logging.error(error) - timestamp = None - finally: - return (nickname, ID, descDigest, timestamp, ORaddr, ORport, dirport) - -@deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -def parseALine(line, fingerprint=None): - """Parse an 'a'-line of a bridge networkstatus document. - - From torspec.git/dir-spec.txt, commit 36761c7d553d L1499-1512: - | - | "a" SP address ":" port NL - | - | [Any number.] - | - | Present only if the OR has at least one IPv6 address. - | - | Address and portlist are as for "or-address" as specified in - | 2.1. - | - | (Only included when the vote or consensus is generated with - | consensus-method 14 or later.) - - :param string line: An 'a'-line from an bridge-network-status descriptor. - :type fingerprint: string or None - :param fingerprint: A string which identifies which OR the descriptor - we're parsing came from (since the 'a'-line doesn't tell us, this can - help make the log messages clearer). - :raises: :exc:`NetworkstatusParsingError` - :rtype: tuple - :returns: A 2-tuple of a string respresenting the IP address and a - :class:`bridgedb.parse.addr.PortList`. - """ - ip = None - portlist = None - - if line.startswith('a '): - line = line[2:] # Chop off the 'a ' - else: - logging.warn("Networkstatus parser received non 'a'-line for %r:"\ - " %r" % (fingerprint or 'Unknown', line)) - - try: - ip, portlist = line.rsplit(':', 1) - except ValueError as error: - logging.error("Bad separator in networkstatus 'a'-line: %r" % line) - return (None, None) - - if ip.startswith('[') and ip.endswith(']'): - ip = ip.strip('[]') - - try: - if not addr.isIPAddress(ip): - raise NetworkstatusParsingError( - "Got invalid IP Address in networkstatus 'a'-line for %r: %r" - % (fingerprint or 'Unknown', line)) - - if addr.isIPv4(ip): - warnings.warn(FutureWarning( - "Got IPv4 address in networkstatus 'a'-line! "\ - "Networkstatus document format may have changed!")) - except NetworkstatusParsingError as error: - logging.error(error) - ip, portlist = None, None - - try: - portlist = addr.PortList(portlist) - if not portlist: - raise NetworkstatusParsingError( - "Got invalid portlist in 'a'-line for %r!\n Line: %r" - % (fingerprint or 'Unknown', line)) - except (addr.InvalidPort, NetworkstatusParsingError) as error: - logging.error(error) - portlist = None - else: - logging.debug("Parsed networkstatus ORAddress line for %r:"\ - "\n Address: %s \tPorts: %s" - % (fingerprint or 'Unknown', ip, portlist)) - finally: - return (ip, portlist) - -@deprecate.deprecated(Version('bridgedb', 0, 2, 5)) -def parseSLine(line): - """Parse an 's'-line from a bridge networkstatus document. - - The 's'-line contains all flags assigned to a bridge. The flags which may - be assigned to a bridge are as follows: - - From torspec.git/dir-spec.txt, commit 36761c7d553d L1526-1554: - | - | "s" SP Flags NL - | - | [Exactly once.] - | - | A series of space-separated status flags, in lexical order (as ASCII - | byte strings). Currently documented flags are: - | - | "BadDirectory" if the router is believed to be useless as a - | directory cache (because its directory port isn't working, - | its bandwidth is always throttled, or for some similar - | reason). - | "Fast" if the router is suitable for high-bandwidth circuits. - | "Guard" if the router is suitable for use as an entry guard. - | "HSDir" if the router is considered a v2 hidden service directory. - | "Named" if the router's identity-nickname mapping is canonical, - | and this authority binds names. - | "Stable" if the router is suitable for long-lived circuits. - | "Running" if the router is currently usable. - | "Valid" if the router has been 'validated'. - | "V2Dir" if the router implements the v2 directory protocol. - - :param string line: An 's'-line from an bridge-network-status descriptor. - :rtype: tuple - :returns: A 2-tuple of booleans, the first is True if the bridge has the - "Running" flag, and the second is True if it has the "Stable" flag. - """ - line = line[2:] - - flags = [x.capitalize() for x in line.split()] - fast = 'Fast' in flags - running = 'Running' in flags - stable = 'Stable' in flags - guard = 'Guard' in flags - valid = 'Valid' in flags - - if (fast or running or stable or guard or valid): - logging.debug("Parsed Flags: %s%s%s%s%s" - % ('Fast ' if fast else '', - 'Running ' if running else '', - 'Stable ' if stable else '', - 'Guard ' if guard else '', - 'Valid ' if valid else '')) - - # Right now, we only care about 'Running' and 'Stable' - return running, stable diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index a039f91..571b3ca 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -34,7 +34,6 @@ from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.Stability import BridgeHistory
from bridgedb.parse import addr -from bridgedb.test import deprecated_networkstatus as networkstatus from bridgedb.test.util import bracketIPv6 from bridgedb.test.util import randomIP from bridgedb.test.util import randomIPv4 diff --git a/lib/bridgedb/test/test_Tests.py b/lib/bridgedb/test/test_Tests.py index e78a4c4..84c3f8b 100644 --- a/lib/bridgedb/test/test_Tests.py +++ b/lib/bridgedb/test/test_Tests.py @@ -84,17 +84,6 @@ def monkeypatchTests(): patches from :mod:`bridgedb.test.deprecated`. """ patcher = monkey.MonkeyPatcher() - patcher.addPatch(Tests.networkstatus, 'parseALine', - deprecated.parseORAddressLine) - patcher.addPatch(Tests.addr, 'PortList', deprecated.PortList) - patcher.addPatch(Tests.bridgedb.Bridges, 'PORTSPEC_LEN', 16) - patcher.addPatch(Tests.bridgedb.Bridges, 're_ipv4', deprecated.re_ipv4) - patcher.addPatch(Tests.bridgedb.Bridges, 're_ipv6', deprecated.re_ipv6) - patcher.addPatch(Tests.bridgedb.Bridges, 'HEX_FP_LEN', 40) - patcher.addPatch(Tests.bridgedb.Bridges, 'toHex', binascii.b2a_hex) - patcher.addPatch(Tests.bridgedb.Bridges, 'fromHex', binascii.a2b_hex) - patcher.addPatch(Tests.bridgedb.Bridges, 'is_valid_fingerprint', - deprecated.is_valid_fingerprint) patcher.addPatch(Tests.bridgedb.Bridges, 'PluggableTransport', deprecated.PluggableTransport) patcher.addPatch(Tests.bridgedb.Bridges, 'Bridge',