March 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Add unittests for the new bridgedb.bridges.PluggableTransport class.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 9f91454a2e32f4b39aed5bb34d8b8221850c41ba Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 23:16:40 2014 +0000 Add unittests for the new bridgedb.bridges.PluggableTransport class. --- lib/bridgedb/test/test_bridges.py | 168 ++++++++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index 6ca96de..2805c6d 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -81,7 +81,7 @@ class BridgeIntegrationTests(unittest.TestCase): b = bridges.Bridge(self.nickname, self.ip, self.orport, fingerprint=self.fingerprint) self.assertIsInstance(b, bridges.Bridge) - + def test_integration_init_1(self): """Ensure that we can initialise the new :class:`bridgedb.bridges.Bridge` class in the same manner as the old :class:`bridgedb.Bridges.Bridge` @@ -91,7 +91,7 @@ class BridgeIntegrationTests(unittest.TestCase): b = bridges.Bridge(self.nickname, self.ip, self.orport, id_digest=self.id_digest) self.assertIsInstance(b, bridges.Bridge) - + def test_integration_init_2(self): """Initialisation of a :class:`bridgedb.bridges.Bridge` with a bad ``id_digest`` should raise a TypeError. @@ -324,3 +324,167 @@ class FlagsTests(unittest.TestCase): self.flags.update(["Fast", "Stable"]) self.assertTrue(self.flags.fast) self.assertTrue(self.flags.stable) + + +class PluggableTransportTests(unittest.TestCase): + """Tests for :class:`bridgedb.bridges.PluggableTransport.""" + + def setUp(self): + self.fingerprint = "ABCDEF0123456789ABCDEF0123456789ABCDEF01" + + def test_PluggableTransport_init_with_parameters(self): + """Initialising a PluggableTransport with args should work.""" + pt = bridges.PluggableTransport(self.fingerprint, + "voltronPT", "1.2.3.4", 443, + {'sharedsecret': 'foobar'}) + self.assertIsInstance(pt, bridges.PluggableTransport) + + def test_PluggableTransport_init(self): + """Initialising a PluggableTransport without args should work.""" + pt = bridges.PluggableTransport() + self.assertIsInstance(pt, bridges.PluggableTransport) + + def test_PluggableTransport_parseArgumentsIntoDict_valid_list(self): + """Parsing a valid list of PT args should return a dictionary.""" + pt = bridges.PluggableTransport() + args = pt._parseArgumentsIntoDict(["sharedsecret=foobar", + "publickey=1234"]) + self.assertIsInstance(args, dict) + self.assertItemsEqual(args, {"sharedsecret": "foobar", + "publickey": "1234"}) + + def test_PluggableTransport_parseArgumentsIntoDict_valid_list_multi(self): + """Parsing a valid list with multiple PT args in a single list element + should return a dictionary. + """ + pt = bridges.PluggableTransport() + args = pt._parseArgumentsIntoDict(["sharedsecret=foobar,password=baz", + "publickey=1234"]) + self.assertIsInstance(args, dict) + self.assertItemsEqual(args, {"sharedsecret": "foobar", + "password": "baz", + "publickey": "1234"}) + + def test_PluggableTransport_parseArgumentsIntoDict_invalid_missing_equals(self): + """Parsing a string of PT args where one PT arg (K=V) is missing an + ``=`` character should raise a ValueError. + """ + pt = bridges.PluggableTransport() + args = pt._parseArgumentsIntoDict( + ["sharedsecret=foobar,password,publickey=1234"]) + self.assertItemsEqual(args, {"sharedsecret": "foobar", + "publickey": "1234"}) + + def test_PluggableTransport_runChecks_invalid_fingerprint(self): + """Calling _runChecks() on a PluggableTransport with an invalid + fingerprint should raise a MalformedPluggableTransport exception. + """ + pt = bridges.PluggableTransport() + self.assertRaises( + bridges.MalformedPluggableTransport, + pt.updateFromStemTransport, + "INVALIDFINGERPRINT", 'obfs4', ('34.230.223.87', 37341, [ + ('iat-mode=0,' + 'node-id=2a79f14120945873482b7823caabe2fcde848722,' + 'public-key=0a5b046d07f6f971b7776de682f57c5b9cdc8fa060db7ef59de82e721c8098f4')])) + + def test_PluggableTransport_runChecks_invalid_ip(self): + """Calling _runChecks() on a PluggableTransport with an invalid + IP address should raise a InvalidPluggableTransportIP exception. + """ + pt = bridges.PluggableTransport() + self.assertRaises( + bridges.InvalidPluggableTransportIP, + pt.updateFromStemTransport, + self.fingerprint, 'obfs4', ('34.230.223', 37341, [ + ('iat-mode=0,' + 'node-id=2a79f14120945873482b7823caabe2fcde848722,')])) + + def test_PluggableTransport_runChecks_invalid_port_type(self): + """Calling _runChecks() on a PluggableTransport with an invalid port + should raise a MalformedPluggableTransport exception. + """ + pt = bridges.PluggableTransport() + self.assertRaises( + bridges.MalformedPluggableTransport, + pt.updateFromStemTransport, + self.fingerprint, 'obfs4', ('34.230.223.87', "anyport", [ + ('iat-mode=0,' + 'node-id=2a79f14120945873482b7823caabe2fcde848722,')])) + + def test_PluggableTransport_runChecks_invalid_port_range(self): + """Calling _runChecks() on a PluggableTransport with an invalid port + (too high) should raise a MalformedPluggableTransport exception. + """ + pt = bridges.PluggableTransport() + self.assertRaises( + bridges.MalformedPluggableTransport, + pt.updateFromStemTransport, + self.fingerprint, 'obfs4', ('34.230.223.87', 65536, [ + ('iat-mode=0,' + 'node-id=2a79f14120945873482b7823caabe2fcde848722,')])) + + def test_PluggableTransport_runChecks_invalid_pt_args(self): + """Calling _runChecks() on a PluggableTransport with an invalid PT + args should raise a MalformedPluggableTransport exception. + """ + try: + pt = bridges.PluggableTransport(self.fingerprint, + "voltronPT", "1.2.3.4", 443, + 'sharedsecret=foobar') + except Exception as error: + self.failUnlessIsInstance(error, + bridges.MalformedPluggableTransport) + + def test_PluggableTransport_getTransportLine_bridge_prefix(self): + """If the 'Bridge ' prefix was requested, then it should be at the + beginning of the bridge line. + """ + pt = bridges.PluggableTransport(self.fingerprint, + "voltronPT", "1.2.3.4", 443, + {'sharedsecret': 'foobar', + 'password': 'unicorns'}) + bridgeLine = pt.getTransportLine(bridgePrefix=True) + self.assertTrue(bridgeLine.startswith("Bridge ")) + + def test_PluggableTransport_getTransportLine_without_Fingerprint(self): + """If no fingerprint was requested, then there shouldn't be a + fingerprint in the bridge line. + """ + pt = bridges.PluggableTransport(self.fingerprint, + "voltronPT", "1.2.3.4", 443, + {'sharedsecret': 'foobar', + 'password': 'unicorns'}) + bridgeLine = pt.getTransportLine(includeFingerprint=False) + self.assertNotSubstring(self.fingerprint, bridgeLine) + + def test_PluggableTransport_getTransportLine_content_order(self): + """Check the order and content of the bridge line string.""" + pt = bridges.PluggableTransport(self.fingerprint, + "voltronPT", "1.2.3.4", 443, + {'sharedsecret': 'foobar', + 'password': 'unicorns'}) + bridgeLine = pt.getTransportLine() + + # We have to check for substrings because we don't know which order + # the PT arguments will end up in the bridge line. Fortunately, the + # following three are the only ones which are important to have in + # order: + self.assertTrue(bridgeLine.startswith("voltronPT")) + self.assertSubstring("voltronPT 1.2.3.4:443 " + self.fingerprint, + bridgeLine) + # These ones can be in any order, but they should be at the end of the + # bridge line: + self.assertSubstring("password=unicorns", bridgeLine) + self.assertSubstring("sharedsecret=foobar", bridgeLine) + + def test_PluggableTransport_getTransportLine_ptargs_space_delimited(self): + """The PT arguments in a bridge line should be space-separated.""" + pt = bridges.PluggableTransport(self.fingerprint, + "voltronPT", "1.2.3.4", 443, + {'sharedsecret': 'foobar', + 'password': 'unicorns'}) + bridgeLine = pt.getTransportLine() + self.assertTrue( + ("password=unicorns sharedsecret=foobar" in bridgeLine) or + ("sharedsecret=foobar password=unicorns" in bridgeLine))

1 0

[bridgedb/master] Add unittest for parsing empty extrainfo files.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit d3e867231019a7401e7f6563833cc1e52f928d8d Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 09:35:04 2014 +0000 Add unittest for parsing empty extrainfo files. --- lib/bridgedb/test/test_parse_descriptors.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/bridgedb/test/test_parse_descriptors.py b/lib/bridgedb/test/test_parse_descriptors.py index b017dd6..0b81dd7 100644 --- a/lib/bridgedb/test/test_parse_descriptors.py +++ b/lib/bridgedb/test/test_parse_descriptors.py @@ -530,3 +530,9 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertRaises(AttributeError, descriptors.parseBridgeExtraInfoFiles, descFileOne, descFileTwo, descFileThree) + + def test_parse_descriptosrs_parseBridgeExtraInfoFiles_empty_file(self): + """Test parsing an empty extrainfo descriptors file.""" + routers = descriptors.parseBridgeExtraInfoFiles(io.BytesIO('')) + self.assertIsInstance(routers, dict) + self.assertEqual(len(routers), 0)

1 0

[bridgedb/master] Move bridgedb.Tests → bridgedb.test.legacy_Tests.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit e6ebfaa1c70f89bee6b5c6f59ab457bc97217b88 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 23:20:55 2014 +0000 Move bridgedb.Tests → bridgedb.test.legacy_Tests. * MOVE bridgedb.Tests to bridgedb.test.legacy_Tests. * CHANGE bridgedb.test.test_Tests to import the moved module. --- lib/bridgedb/Tests.py | 780 ------------------------------------ lib/bridgedb/test/legacy_Tests.py | 785 +++++++++++++++++++++++++++++++++++++ lib/bridgedb/test/test_Tests.py | 9 +- 3 files changed, 790 insertions(+), 784 deletions(-) diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py deleted file mode 100644 index 93ee4f2..0000000 --- a/lib/bridgedb/Tests.py +++ /dev/null @@ -1,780 +0,0 @@ -# BridgeDB by Nick Mathewson. -# Copyright (c) 2007-2009, The Tor Project, Inc. -# See LICENSE for licensing information - -from __future__ import print_function - -import doctest -import os -import random -import sqlite3 -import tempfile -import unittest -import warnings -import time -from datetime import datetime - -import bridgedb.Bridges -import bridgedb.Main -import bridgedb.Dist -import bridgedb.schedule -import bridgedb.Storage -import re -import ipaddr - -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByOnlyIP4 -from bridgedb.Filters import filterBridgesByOnlyIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn - -from bridgedb.Stability import BridgeHistory - -from bridgedb.parse import addr -from bridgedb.parse import networkstatus - -from math import log - -def suppressWarnings(): - warnings.filterwarnings('ignore', '.*tmpnam.*') - -def randomIP(): - if random.choice(xrange(2)): - return randomIP4() - return randomIP6() - -def randomIP4(): - return ipaddr.IPv4Address(random.getrandbits(32)) - -def randomIP4String(): - return randomIP4().compressed - -def randomIP6(): - return ipaddr.IPv6Address(random.getrandbits(128)) - -def randomIP6String(): - return bracketIP6(randomIP6().compressed) - -def randomIPString(): - if random.choice(xrange(2)): - return randomIP4String() - return randomIP6String() - -def bracketIP6(ip): - """Put brackets around an IPv6 address, just as tor does.""" - return "[%s]" % ip - -def random16IP(): - upper = "123.123." # same 16 - lower = ".".join([str(random.randrange(1,256)) for _ in xrange(2)]) - return upper+lower - -def randomPort(): - return random.randint(1,65535) - -def randomPortSpec(): - """ - returns a random list of ports - """ - ports = [] - for i in range(0,24): - ports.append(random.randint(1,65535)) - ports.sort(reverse=True) - - portspec = "" - for i in range(0,16): - portspec += "%d," % random.choice(ports) - portspec = portspec.rstrip(',') #remove trailing , - return portspec - -def randomCountry(): - countries = ['us', 'nl', 'de', 'cz', 'sk', 'as', 'si', 'it'] - #XXX: load from geoip - return random.choice(countries) - -def randomCountrySpec(): - countries = ['us', 'nl', 'de', 'cz', 'sk', 'as', 'si', 'it'] - #XXX: load from geoip - spec = "" - choices = [] - for i in xrange(10): - choices.append(random.choice(countries)) - choices = set(choices) #dedupe - choices = list(choices) - spec += ",".join(choices) - return spec - -def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, - transports=False): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = ipaddr.IPAddress(randomIP4()) - fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) - b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) - b.setStatus(running, stable) - - oraddrs = [] - if or_addresses: - for i in xrange(8): - # Only add or_addresses if they are valid. Otherwise, the test - # will randomly fail if an invalid address is chosen: - address = randomIP4String() - portlist = addr.PortList(randomPortSpec()) - if addr.isValidIP(address): - oraddrs.append((address, portlist,)) - - for address, portlist in oraddrs: - networkstatus.parseALine("{0}:{1}".format(address, portlist)) - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist - - if transports: - for i in xrange(0,8): - b.transports.append(bridgedb.Bridges.PluggableTransport(b, - random.choice(["obfs", "obfs2", "pt1"]), - randomIP(), randomPort())) - return b - -def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, - transports=False): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = ipaddr.IPAddress(randomIP6()) - fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) - b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) - b.setStatus(running, stable) - - oraddrs = [] - if or_addresses: - for i in xrange(8): - # Only add or_addresses if they are valid. Otherwise, the test - # will randomly fail if an invalid address is chosen: - address = randomIP6() - portlist = addr.PortList(randomPortSpec()) - if addr.isValidIP(address): - address = bracketIP6(address) - oraddrs.append((address, portlist,)) - - for address, portlist in oraddrs: - networkstatus.parseALine("{0}:{1}".format(address, portlist)) - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist - - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist - - if transports: - for i in xrange(0,8): - b.transports.append(bridgedb.Bridges.PluggableTransport(b, - random.choice(["obfs", "obfs2", "pt1"]), - randomIP(), randomPort())) - - return b - -def fake16Bridge(orport=8080, running=True, stable=True): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = random16IP() - fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) - b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) - b.setStatus(running, stable) - return b - -simpleDesc = "router Unnamed %s %s 0 9030\n"\ -"opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ -"opt @purpose bridge\n" -orAddress = "or-address %s:%s\n" -def gettimestamp(): - ts = time.strftime("%Y-%m-%d %H:%M:%S") - return "opt published %s\n" % ts - -class RhymesWith255Category: - def contains(self, ip): - return ip.endswith(".255") - -class EmailBridgeDistTests(unittest.TestCase): - def setUp(self): - self.fd, self.fname = tempfile.mkstemp() - self.db = bridgedb.Storage.Database(self.fname) - bridgedb.Storage.setDB(self.db) - self.cur = self.db._conn.cursor() - - def tearDown(self): - self.db.close() - os.close(self.fd) - os.unlink(self.fname) - - def testEmailRateLimit(self): - db = self.db - EMAIL_DOMAIN_MAP = {'example.com':'example.com'} - d = bridgedb.Dist.EmailBasedDistributor( - "Foo", - {'example.com': 'example.com', - 'dkim.example.com': 'dkim.example.com'}, - {'example.com': [], 'dkim.example.com': ['dkim']}) - for _ in xrange(256): - d.insert(fakeBridge()) - d.getBridgesForEmail('abc(a)example.com', 1, 3) - self.assertRaises(bridgedb.Dist.TooSoonEmail, - d.getBridgesForEmail, 'abc(a)example.com', 1, 3) - self.assertRaises(bridgedb.Dist.IgnoreEmail, - d.getBridgesForEmail, 'abc(a)example.com', 1, 3) - - def testUnsupportedDomain(self): - db = self.db - self.assertRaises(bridgedb.parse.addr.UnsupportedDomain, - bridgedb.parse.addr.normalizeEmail, - 'bad(a)email.com', - {'example.com':'example.com'}, - {'example.com':[]}) - -class IPBridgeDistTests(unittest.TestCase): - def dumbAreaMapper(self, ip): - return ip - def testBasicDist(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - for _ in xrange(256): - d.insert(fakeBridge()) - n = d.getBridgesForIP("1.2.3.4", "x", 2) - n2 = d.getBridgesForIP("1.2.3.4", "x", 2) - self.assertEquals(n, n2) - - def testDistWithCategories(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo", - [RhymesWith255Category()]) - assert len(d.categories) == 1 - for _ in xrange(256): - d.insert(fakeBridge()) - - for _ in xrange(256): - # Make sure that the categories do not overlap - f = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(4)]) - g = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(3)] + ['255']) - n = d.getBridgesForIP(g(), "x", 10) - n2 = d.getBridgesForIP(f(), "x", 10) - - assert(len(n) > 0) - assert(len(n2) > 0) - - for b in n: - assert (b not in n2) - - for b in n2: - assert (b not in n) - - #XXX: #6175 breaks this test! - #def testDistWithPortRestrictions(self): - # param = bridgedb.Bridges.BridgeRingParameters(needPorts=[(443, 1)]) - # d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Baz", - # answerParameters=param) - # for _ in xrange(32): - # d.insert(fakeBridge(443)) - # for _ in range(256): - # d.insert(fakeBridge()) - # for _ in xrange(32): - # i = randomIP() - # n = d.getBridgesForIP(i, "x", 5) - # count = 0 - # fps = {} - # for b in n: - # fps[b.getID()] = 1 - # if b.orport == 443: - # count += 1 - # self.assertEquals(len(fps), len(n)) - # self.assertEquals(len(fps), 5) - # self.assertTrue(count >= 1) - - #def testDistWithFilter16(self): - # d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - # for _ in xrange(256): - # d.insert(fake16Bridge()) - # n = d.getBridgesForIP("1.2.3.4", "x", 10) - - # slash16s = dict() - # for bridge in n: - # m = re.match(r'(\d+\.\d+)\.\d+\.\d+', bridge.ip) - # upper16 = m.group(1) - # self.assertTrue(upper16 not in slash16s) - # slash16s[upper16] = True - - def testDistWithFilterIP6(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for i in xrange(500): - bridges = d.getBridgesForIP(randomIP4String(), - "faketimestamp", - bridgeFilterRules=[filterBridgesByIP6]) - bridge = random.choice(bridges) - bridge_line = bridge.getConfigLine(addressClass=ipaddr.IPv6Address) - address, portlist = networkstatus.parseALine(bridge_line) - assert type(ipaddr.IPAddress(address)) is ipaddr.IPv6Address - assert filterBridgesByIP6(random.choice(bridges)) - - def testDistWithFilterIP4(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for i in xrange(500): - bridges = d.getBridgesForIP(randomIP4String(), - "faketimestamp", - bridgeFilterRules=[filterBridgesByIP4]) - bridge = random.choice(bridges) - bridge_line = bridge.getConfigLine(addressClass=ipaddr.IPv4Address) - address, portlist = networkstatus.parseALine(bridge_line) - assert type(ipaddr.IPAddress(address)) is ipaddr.IPv4Address - assert filterBridgesByIP4(random.choice(bridges)) - - def testDistWithFilterBoth(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for i in xrange(50): - bridges = d.getBridgesForIP(randomIP4String(), - "faketimestamp", 1, - bridgeFilterRules=[ - filterBridgesByIP4, - filterBridgesByIP6]) - if bridges: - t = bridges.pop() - assert filterBridgesByIP4(t) - assert filterBridgesByIP6(t) - address, portlist = networkstatus.parseALine( - t.getConfigLine(addressClass=ipaddr.IPv4Address)) - assert type(address) is ipaddr.IPv4Address - address, portlist = networkstatus.parseALine( - t.getConfigLine(addressClass=ipaddr.IPv6Address)) - assert type(address) is ipaddr.IPv6Address - - - def testDistWithFilterAll(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for i in xrange(5): - b = d.getBridgesForIP(randomIP4String(), "x", 1, bridgeFilterRules=[ - filterBridgesByOnlyIP4, filterBridgesByOnlyIP6]) - assert len(b) == 0 - - def testDistWithFilterBlockedCountries(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for b in d.splitter.bridges: - # china blocks all :-( - for pt in b.transports: - key = "%s:%s" % (pt.address, pt.port) - b.blockingCountries[key] = set(['cn']) - for address, portlist in b.or_addresses.items(): - for port in portlist: - key = "%s:%s" % (address, port) - b.blockingCountries[key] = set(['cn']) - key = "%s:%s" % (b.ip, b.orport) - b.blockingCountries[key] = set(['cn']) - - for i in xrange(5): - b = d.getBridgesForIP(randomIP4String(), "x", 1, bridgeFilterRules=[ - filterBridgesByNotBlockedIn("cn")]) - assert len(b) == 0 - b = d.getBridgesForIP(randomIP4String(), "x", 1, bridgeFilterRules=[ - filterBridgesByNotBlockedIn("us")]) - assert len(b) > 0 - - def testDistWithFilterBlockedCountriesAdvanced(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True, transports=True)) - d.insert(fakeBridge(or_addresses=True, transports=True)) - - for b in d.splitter.bridges: - # china blocks some transports - for pt in b.transports: - if random.choice(xrange(2)) > 0: - key = "%s:%s" % (pt.address, pt.port) - b.blockingCountries[key] = set(['cn']) - for address, portlist in b.or_addresses.items(): - # china blocks some transports - for port in portlist: - if random.choice(xrange(2)) > 0: - key = "%s:%s" % (address, port) - b.blockingCountries[key] = set(['cn']) - key = "%s:%s" % (b.ip, b.orport) - b.blockingCountries[key] = set(['cn']) - - # we probably will get at least one bridge back! - # it's pretty unlikely to lose a coin flip 250 times in a row - for i in xrange(5): - b = d.getBridgesForIP(randomIPString(), "x", 1, - bridgeFilterRules=[ - filterBridgesByNotBlockedIn("cn", methodname='obfs2'), - filterBridgesByTransport('obfs2'), - ]) - try: assert len(b) > 0 - except AssertionError: - print("epic fail") - b = d.getBridgesForIP(randomIPString(), "x", 1, bridgeFilterRules=[ - filterBridgesByNotBlockedIn("us")]) - assert len(b) > 0 - - -class SQLStorageTests(unittest.TestCase): - def setUp(self): - self.fd, self.fname = tempfile.mkstemp() - self.db = bridgedb.Storage.Database(self.fname) - self.cur = self.db._conn.cursor() - - def tearDown(self): - self.db.close() - os.close(self.fd) - os.unlink(self.fname) - - def assertCloseTo(self, a, b, delta=60): - self.assertTrue(abs(a-b) <= delta) - - def testBridgeStorage(self): - db = self.db - B = bridgedb.Bridges.Bridge - t = time.time() - cur = self.cur - - k1 = "aaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbb" - k2 = "abababababababababababababababababababab" - k3 = "cccccccccccccccccccccccccccccccccccccccc" - b1 = B("serv1", "1.2.3.4", 999, fingerprint=k1) - b1_v2 = B("serv1", "1.2.3.5", 9099, fingerprint=k1) - b2 = B("serv2", "2.3.4.5", 9990, fingerprint=k2) - b3 = B("serv3", "2.3.4.6", 9008, fingerprint=k3) - validRings = ["ring1", "ring2", "ring3"] - - r = db.insertBridgeAndGetRing(b1, "ring1", t, validRings) - self.assertEquals(r, "ring1") - r = db.insertBridgeAndGetRing(b1, "ring10", t+500, validRings) - self.assertEquals(r, "ring1") - - cur.execute("SELECT distributor, address, or_port, first_seen, " - "last_seen FROM Bridges WHERE hex_key = ?", (k1,)) - v = cur.fetchone() - self.assertEquals(v, - ("ring1", "1.2.3.4", 999, - bridgedb.Storage.timeToStr(t), - bridgedb.Storage.timeToStr(t+500))) - - r = db.insertBridgeAndGetRing(b1_v2, "ring99", t+800, validRings) - self.assertEquals(r, "ring1") - cur.execute("SELECT distributor, address, or_port, first_seen, " - "last_seen FROM Bridges WHERE hex_key = ?", (k1,)) - v = cur.fetchone() - self.assertEquals(v, - ("ring1", "1.2.3.5", 9099, - bridgedb.Storage.timeToStr(t), - bridgedb.Storage.timeToStr(t+800))) - - db.insertBridgeAndGetRing(b2, "ring2", t, validRings) - db.insertBridgeAndGetRing(b3, "ring3", t, validRings) - - cur.execute("SELECT COUNT(distributor) FROM Bridges") - v = cur.fetchone() - self.assertEquals(v, (3,)) - - r = db.getEmailTime("abc(a)example.com") - self.assertEquals(r, None) - db.setEmailTime("abc(a)example.com", t) - db.setEmailTime("def(a)example.com", t+1000) - r = db.getEmailTime("abc(a)example.com") - self.assertCloseTo(r, t) - r = db.getEmailTime("def(a)example.com") - self.assertCloseTo(r, t+1000) - r = db.getEmailTime("ghi(a)example.com") - self.assertEquals(r, None) - - db.cleanEmailedBridges(t+200) - db.setEmailTime("def(a)example.com", t+5000) - r = db.getEmailTime("abc(a)example.com") - self.assertEquals(r, None) - r = db.getEmailTime("def(a)example.com") - self.assertCloseTo(r, t+5000) - cur.execute("SELECT * FROM EmailedBridges") - self.assertEquals(len(cur.fetchall()), 1) - - db.addBridgeBlock(b2.fingerprint, 'us') - self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), True) - db.delBridgeBlock(b2.fingerprint, 'us') - self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), False) - db.addBridgeBlock(b2.fingerprint, 'uk') - db.addBridgeBlock(b3.fingerprint, 'uk') - self.assertEquals(set([b2.fingerprint, b3.fingerprint]), - set(db.getBlockedBridges('uk'))) - - db.addBridgeBlock(b2.fingerprint, 'cn') - db.addBridgeBlock(b2.fingerprint, 'de') - db.addBridgeBlock(b2.fingerprint, 'jp') - db.addBridgeBlock(b2.fingerprint, 'se') - db.addBridgeBlock(b2.fingerprint, 'kr') - - self.assertEquals(set(db.getBlockingCountries(b2.fingerprint)), - set(['uk', 'cn', 'de', 'jp', 'se', 'kr'])) - self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) - db.setWarnedEmail("def(a)example.com") - self.assertEquals(db.getWarnedEmail("def(a)example.com"), True) - db.setWarnedEmail("def(a)example.com", False) - self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) - - db.setWarnedEmail("def(a)example.com") - self.assertEquals(db.getWarnedEmail("def(a)example.com"), True) - db.cleanWarnedEmails(t+200) - self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) - -class ParseDescFileTests(unittest.TestCase): - def testSimpleDesc(self): - test = "" - - for i in range(100): - test+= "".join(simpleDesc % (randomIP(), randomPort())) - test+=gettimestamp() - test+="router-signature\n" - - bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] - self.assertEquals(len(bs), 100) - - for b in bs: - b.assertOK() - - def testSingleOrAddress(self): - test = "" - - for i in range(100): - test+= simpleDesc % (randomIP(), randomPort()) - test+= orAddress % (randomIP(),randomPort()) - test+=gettimestamp() - test+= "router-signature\n" - - bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] - self.assertEquals(len(bs), 100) - - for b in bs: - b.assertOK() - - def testMultipleOrAddress(self): - test = "" - for i in range(100): - test+= simpleDesc % (randomIPString(), randomPort()) - for i in xrange(8): - test+= orAddress % (randomIPString(),randomPortSpec()) - test+=gettimestamp() - test+= "router-signature\n" - - bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] - self.assertEquals(len(bs), 100) - - for b in bs: - b.assertOK() - - def testConvolutedOrAddress(self): - test = "" - for i in range(100): - test+= simpleDesc % (randomIPString(), randomPort()) - for i in xrange(8): - test+= orAddress % (randomIPString(),randomPortSpec()) - test+=gettimestamp() - test+= "router-signature\n" - - bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] - self.assertEquals(len(bs), 100) - - for b in bs: - b.assertOK() - - def testParseCountryBlockFile(self): - simpleBlock = "%s:%s %s\n" - countries = ['us', 'nl', 'de', 'cz', 'sk', 'as', 'si', 'it'] - test = str() - for i in range(100): - test += simpleBlock % (randomIPString(), randomPort(), - randomCountrySpec()) - test+=gettimestamp() - - for a,p,c in bridgedb.Bridges.parseCountryBlockFile(test.split('\n')): - assert type(a) in (ipaddr.IPv6Address, ipaddr.IPv4Address) - assert isinstance(p, addr.PortList) - assert isinstance(c, list) - assert len(c) > 0 - for y in c: - assert y in countries - #print "address: %s" % a - #print "portlist: %s" % p - #print "countries: %s" % c - -class BridgeStabilityTests(unittest.TestCase): - def setUp(self): - self.fd, self.fname = tempfile.mkstemp() - self.db = bridgedb.Storage.Database(self.fname) - bridgedb.Storage.setDB(self.db) - self.cur = self.db._conn.cursor() - - def tearDown(self): - self.db.close() - os.close(self.fd) - os.unlink(self.fname) - - def testAddOrUpdateSingleBridgeHistory(self): - db = self.db - b = fakeBridge() - timestamp = time.time() - bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b, timestamp) - assert isinstance(bhe, BridgeHistory) - assert isinstance(db.getBridgeHistory(b.fingerprint), BridgeHistory) - assert len([y for y in db.getAllBridgeHistory()]) == 1 - - def testDeletingSingleBridgeHistory(self): - db = self.db - b = fakeBridge() - timestamp = time.time() - bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b, timestamp) - assert isinstance(bhe, BridgeHistory) - assert isinstance(db.getBridgeHistory(b.fingerprint), BridgeHistory) - db.delBridgeHistory(b.fingerprint) - assert db.getBridgeHistory(b.fingerprint) is None - assert len([y for y in db.getAllBridgeHistory()]) == 0 - - def testTOSA(self): - db = self.db - b = random.choice([fakeBridge,fakeBridge6])() - def timestampSeries(x): - for i in xrange(61): - yield (i+1)*60*30 + x # 30 minute intervals - now = time.time() - time_on_address = long(60*30*60) # 30 hours - downtime = 60*60*random.randint(0,4) # random hours of downtime - - for t in timestampSeries(now): - bridgedb.Stability.addOrUpdateBridgeHistory(b,t) - assert db.getBridgeHistory(b.fingerprint).tosa == time_on_address - - b.orport += 1 - - for t in timestampSeries(now + time_on_address + downtime): - bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b,t) - assert db.getBridgeHistory(b.fingerprint).tosa == time_on_address + downtime - - def testLastSeenWithDifferentAddressAndPort(self): - db = self.db - for i in xrange(10): - num_desc = 30 - time_start = time.time() - ts = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] - b = random.choice([fakeBridge(), fakeBridge6()]) - [ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ] - - # change the port - b.orport = b.orport+1 - last_seen = ts[-1] - ts = [ 60*30*(i+1) + last_seen for i in xrange(num_desc) ] - [ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ] - b = db.getBridgeHistory(b.fingerprint) - assert b.tosa == ts[-1] - last_seen - assert (long(last_seen*1000) == b.lastSeenWithDifferentAddressAndPort) - assert (long(ts[-1]*1000) == b.lastSeenWithThisAddressAndPort) - - def testFamiliar(self): - # create some bridges - # XXX: slow - num_bridges = 10 - num_desc = 4*48 # 30m intervals, 48 per day - time_start = time.time() - bridges = [ fakeBridge() for x in xrange(num_bridges) ] - t = time.time() - ts = [ (i+1)*60*30+t for i in xrange(num_bridges) ] - for b in bridges: - time_series = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] - [ bridgedb.Stability.addOrUpdateBridgeHistory(b, i) for i in time_series ] - assert None not in bridges - # +1 to avoid rounding errors - assert bridges[-(num_bridges/8 + 1)].familiar == True - - def testDiscountAndPruneBridgeHistory(self): - """ Test pruning of old Bridge History """ - if os.environ.get('TRAVIS'): - self.skipTest("Hangs on Travis-CI.") - - db = self.db - - # make a bunch of bridges - num_bridges = 20 - time_start = time.time() - bridges = [random.choice([fakeBridge, fakeBridge6])() - for i in xrange(num_bridges)] - - # run some of the bridges for the full time series - running = bridges[:num_bridges/2] - # and some that are not - expired = bridges[num_bridges/2:] - - for b in running: assert b not in expired - - # Solving: - # 1 discount event per 12 hours, 24 descriptors 30m apart - num_successful = random.randint(2,60) - # figure out how many intervals it will take for weightedUptime to - # decay to < 1 - num_desc = int(30*log(1/float(num_successful*30*60))/(-0.05)) - timeseries = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] - - for i in timeseries: - for b in running: - bridgedb.Stability.addOrUpdateBridgeHistory(b, i) - - if num_successful > 0: - for b in expired: - bridgedb.Stability.addOrUpdateBridgeHistory(b, i) - num_successful -= 1 - - # now we expect to see the bridge has been removed from history - for bridge in expired: - b = db.getBridgeHistory(bridge.fingerprint) - assert b is None - # and make sure none of the others have - for bridge in running: - b = db.getBridgeHistory(bridge.fingerprint) - assert b is not None - -def testSuite(): - suite = unittest.TestSuite() - loader = unittest.TestLoader() - - for klass in [IPBridgeDistTests, SQLStorageTests, EmailBridgeDistTests, - ParseDescFileTests, BridgeStabilityTests]: - suite.addTest(loader.loadTestsFromTestCase(klass)) - - for module in [ bridgedb.Bridges, - bridgedb.Main, - bridgedb.Dist, - bridgedb.schedule ]: - suite.addTest(doctest.DocTestSuite(module)) - - return suite - -def main(): - suppressWarnings() - - unittest.TextTestRunner(verbosity=1).run(testSuite()) - diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py new file mode 100644 index 0000000..fa7a13c --- /dev/null +++ b/lib/bridgedb/test/legacy_Tests.py @@ -0,0 +1,785 @@ +# BridgeDB by Nick Mathewson. +# Copyright (c) 2007-2009, The Tor Project, Inc. +# See LICENSE for licensing information + +"""These are legacy integration and unittests which historically lived at +``lib/bridgedb/Tests.py``. They have been moved here to keep the test code +separate from the production codebase. +""" + +from __future__ import print_function + +import doctest +import os +import random +import sqlite3 +import tempfile +import unittest +import warnings +import time +from datetime import datetime + +import bridgedb.Bridges +import bridgedb.Main +import bridgedb.Dist +import bridgedb.schedule +import bridgedb.Storage +import re +import ipaddr + +from bridgedb.Filters import filterBridgesByIP4 +from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.Filters import filterBridgesByOnlyIP4 +from bridgedb.Filters import filterBridgesByOnlyIP6 +from bridgedb.Filters import filterBridgesByTransport +from bridgedb.Filters import filterBridgesByNotBlockedIn + +from bridgedb.Stability import BridgeHistory + +from bridgedb.parse import addr +from bridgedb.parse import networkstatus + +from math import log + +def suppressWarnings(): + warnings.filterwarnings('ignore', '.*tmpnam.*') + +def randomIP(): + if random.choice(xrange(2)): + return randomIP4() + return randomIP6() + +def randomIP4(): + return ipaddr.IPv4Address(random.getrandbits(32)) + +def randomIP4String(): + return randomIP4().compressed + +def randomIP6(): + return ipaddr.IPv6Address(random.getrandbits(128)) + +def randomIP6String(): + return bracketIP6(randomIP6().compressed) + +def randomIPString(): + if random.choice(xrange(2)): + return randomIP4String() + return randomIP6String() + +def bracketIP6(ip): + """Put brackets around an IPv6 address, just as tor does.""" + return "[%s]" % ip + +def random16IP(): + upper = "123.123." # same 16 + lower = ".".join([str(random.randrange(1,256)) for _ in xrange(2)]) + return upper+lower + +def randomPort(): + return random.randint(1,65535) + +def randomPortSpec(): + """ + returns a random list of ports + """ + ports = [] + for i in range(0,24): + ports.append(random.randint(1,65535)) + ports.sort(reverse=True) + + portspec = "" + for i in range(0,16): + portspec += "%d," % random.choice(ports) + portspec = portspec.rstrip(',') #remove trailing , + return portspec + +def randomCountry(): + countries = ['us', 'nl', 'de', 'cz', 'sk', 'as', 'si', 'it'] + #XXX: load from geoip + return random.choice(countries) + +def randomCountrySpec(): + countries = ['us', 'nl', 'de', 'cz', 'sk', 'as', 'si', 'it'] + #XXX: load from geoip + spec = "" + choices = [] + for i in xrange(10): + choices.append(random.choice(countries)) + choices = set(choices) #dedupe + choices = list(choices) + spec += ",".join(choices) + return spec + +def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, + transports=False): + nn = "bridge-%s"%random.randrange(0,1000000) + ip = ipaddr.IPAddress(randomIP4()) + fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) + b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + b.setStatus(running, stable) + + oraddrs = [] + if or_addresses: + for i in xrange(8): + # Only add or_addresses if they are valid. Otherwise, the test + # will randomly fail if an invalid address is chosen: + address = randomIP4String() + portlist = addr.PortList(randomPortSpec()) + if addr.isValidIP(address): + oraddrs.append((address, portlist,)) + + for address, portlist in oraddrs: + networkstatus.parseALine("{0}:{1}".format(address, portlist)) + try: + portlist.add(b.or_addresses[address]) + except KeyError: + pass + finally: + b.or_addresses[address] = portlist + + if transports: + for i in xrange(0,8): + b.transports.append(bridgedb.Bridges.PluggableTransport(b, + random.choice(["obfs", "obfs2", "pt1"]), + randomIP(), randomPort())) + return b + +def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, + transports=False): + nn = "bridge-%s"%random.randrange(0,1000000) + ip = ipaddr.IPAddress(randomIP6()) + fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) + b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + b.setStatus(running, stable) + + oraddrs = [] + if or_addresses: + for i in xrange(8): + # Only add or_addresses if they are valid. Otherwise, the test + # will randomly fail if an invalid address is chosen: + address = randomIP6() + portlist = addr.PortList(randomPortSpec()) + if addr.isValidIP(address): + address = bracketIP6(address) + oraddrs.append((address, portlist,)) + + for address, portlist in oraddrs: + networkstatus.parseALine("{0}:{1}".format(address, portlist)) + try: + portlist.add(b.or_addresses[address]) + except KeyError: + pass + finally: + b.or_addresses[address] = portlist + + try: + portlist.add(b.or_addresses[address]) + except KeyError: + pass + finally: + b.or_addresses[address] = portlist + + if transports: + for i in xrange(0,8): + b.transports.append(bridgedb.Bridges.PluggableTransport(b, + random.choice(["obfs", "obfs2", "pt1"]), + randomIP(), randomPort())) + + return b + +def fake16Bridge(orport=8080, running=True, stable=True): + nn = "bridge-%s"%random.randrange(0,1000000) + ip = random16IP() + fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) + b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + b.setStatus(running, stable) + return b + +simpleDesc = "router Unnamed %s %s 0 9030\n"\ +"opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ +"opt @purpose bridge\n" +orAddress = "or-address %s:%s\n" +def gettimestamp(): + ts = time.strftime("%Y-%m-%d %H:%M:%S") + return "opt published %s\n" % ts + +class RhymesWith255Category: + def contains(self, ip): + return ip.endswith(".255") + +class EmailBridgeDistTests(unittest.TestCase): + def setUp(self): + self.fd, self.fname = tempfile.mkstemp() + self.db = bridgedb.Storage.Database(self.fname) + bridgedb.Storage.setDB(self.db) + self.cur = self.db._conn.cursor() + + def tearDown(self): + self.db.close() + os.close(self.fd) + os.unlink(self.fname) + + def testEmailRateLimit(self): + db = self.db + EMAIL_DOMAIN_MAP = {'example.com':'example.com'} + d = bridgedb.Dist.EmailBasedDistributor( + "Foo", + {'example.com': 'example.com', + 'dkim.example.com': 'dkim.example.com'}, + {'example.com': [], 'dkim.example.com': ['dkim']}) + for _ in xrange(256): + d.insert(fakeBridge()) + d.getBridgesForEmail('abc(a)example.com', 1, 3) + self.assertRaises(bridgedb.Dist.TooSoonEmail, + d.getBridgesForEmail, 'abc(a)example.com', 1, 3) + self.assertRaises(bridgedb.Dist.IgnoreEmail, + d.getBridgesForEmail, 'abc(a)example.com', 1, 3) + + def testUnsupportedDomain(self): + db = self.db + self.assertRaises(bridgedb.parse.addr.UnsupportedDomain, + bridgedb.parse.addr.normalizeEmail, + 'bad(a)email.com', + {'example.com':'example.com'}, + {'example.com':[]}) + +class IPBridgeDistTests(unittest.TestCase): + def dumbAreaMapper(self, ip): + return ip + def testBasicDist(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(256): + d.insert(fakeBridge()) + n = d.getBridgesForIP("1.2.3.4", "x", 2) + n2 = d.getBridgesForIP("1.2.3.4", "x", 2) + self.assertEquals(n, n2) + + def testDistWithCategories(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo", + [RhymesWith255Category()]) + assert len(d.categories) == 1 + for _ in xrange(256): + d.insert(fakeBridge()) + + for _ in xrange(256): + # Make sure that the categories do not overlap + f = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(4)]) + g = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(3)] + ['255']) + n = d.getBridgesForIP(g(), "x", 10) + n2 = d.getBridgesForIP(f(), "x", 10) + + assert(len(n) > 0) + assert(len(n2) > 0) + + for b in n: + assert (b not in n2) + + for b in n2: + assert (b not in n) + + #XXX: #6175 breaks this test! + #def testDistWithPortRestrictions(self): + # param = bridgedb.Bridges.BridgeRingParameters(needPorts=[(443, 1)]) + # d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Baz", + # answerParameters=param) + # for _ in xrange(32): + # d.insert(fakeBridge(443)) + # for _ in range(256): + # d.insert(fakeBridge()) + # for _ in xrange(32): + # i = randomIP() + # n = d.getBridgesForIP(i, "x", 5) + # count = 0 + # fps = {} + # for b in n: + # fps[b.getID()] = 1 + # if b.orport == 443: + # count += 1 + # self.assertEquals(len(fps), len(n)) + # self.assertEquals(len(fps), 5) + # self.assertTrue(count >= 1) + + #def testDistWithFilter16(self): + # d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + # for _ in xrange(256): + # d.insert(fake16Bridge()) + # n = d.getBridgesForIP("1.2.3.4", "x", 10) + + # slash16s = dict() + # for bridge in n: + # m = re.match(r'(\d+\.\d+)\.\d+\.\d+', bridge.ip) + # upper16 = m.group(1) + # self.assertTrue(upper16 not in slash16s) + # slash16s[upper16] = True + + def testDistWithFilterIP6(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) + + for i in xrange(500): + bridges = d.getBridgesForIP(randomIP4String(), + "faketimestamp", + bridgeFilterRules=[filterBridgesByIP6]) + bridge = random.choice(bridges) + bridge_line = bridge.getConfigLine(addressClass=ipaddr.IPv6Address) + address, portlist = networkstatus.parseALine(bridge_line) + assert type(ipaddr.IPAddress(address)) is ipaddr.IPv6Address + assert filterBridgesByIP6(random.choice(bridges)) + + def testDistWithFilterIP4(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) + + for i in xrange(500): + bridges = d.getBridgesForIP(randomIP4String(), + "faketimestamp", + bridgeFilterRules=[filterBridgesByIP4]) + bridge = random.choice(bridges) + bridge_line = bridge.getConfigLine(addressClass=ipaddr.IPv4Address) + address, portlist = networkstatus.parseALine(bridge_line) + assert type(ipaddr.IPAddress(address)) is ipaddr.IPv4Address + assert filterBridgesByIP4(random.choice(bridges)) + + def testDistWithFilterBoth(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) + + for i in xrange(50): + bridges = d.getBridgesForIP(randomIP4String(), + "faketimestamp", 1, + bridgeFilterRules=[ + filterBridgesByIP4, + filterBridgesByIP6]) + if bridges: + t = bridges.pop() + assert filterBridgesByIP4(t) + assert filterBridgesByIP6(t) + address, portlist = networkstatus.parseALine( + t.getConfigLine(addressClass=ipaddr.IPv4Address)) + assert type(address) is ipaddr.IPv4Address + address, portlist = networkstatus.parseALine( + t.getConfigLine(addressClass=ipaddr.IPv6Address)) + assert type(address) is ipaddr.IPv6Address + + + def testDistWithFilterAll(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) + + for i in xrange(5): + b = d.getBridgesForIP(randomIP4String(), "x", 1, bridgeFilterRules=[ + filterBridgesByOnlyIP4, filterBridgesByOnlyIP6]) + assert len(b) == 0 + + def testDistWithFilterBlockedCountries(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) + + for b in d.splitter.bridges: + # china blocks all :-( + for pt in b.transports: + key = "%s:%s" % (pt.address, pt.port) + b.blockingCountries[key] = set(['cn']) + for address, portlist in b.or_addresses.items(): + for port in portlist: + key = "%s:%s" % (address, port) + b.blockingCountries[key] = set(['cn']) + key = "%s:%s" % (b.ip, b.orport) + b.blockingCountries[key] = set(['cn']) + + for i in xrange(5): + b = d.getBridgesForIP(randomIP4String(), "x", 1, bridgeFilterRules=[ + filterBridgesByNotBlockedIn("cn")]) + assert len(b) == 0 + b = d.getBridgesForIP(randomIP4String(), "x", 1, bridgeFilterRules=[ + filterBridgesByNotBlockedIn("us")]) + assert len(b) > 0 + + def testDistWithFilterBlockedCountriesAdvanced(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True, transports=True)) + d.insert(fakeBridge(or_addresses=True, transports=True)) + + for b in d.splitter.bridges: + # china blocks some transports + for pt in b.transports: + if random.choice(xrange(2)) > 0: + key = "%s:%s" % (pt.address, pt.port) + b.blockingCountries[key] = set(['cn']) + for address, portlist in b.or_addresses.items(): + # china blocks some transports + for port in portlist: + if random.choice(xrange(2)) > 0: + key = "%s:%s" % (address, port) + b.blockingCountries[key] = set(['cn']) + key = "%s:%s" % (b.ip, b.orport) + b.blockingCountries[key] = set(['cn']) + + # we probably will get at least one bridge back! + # it's pretty unlikely to lose a coin flip 250 times in a row + for i in xrange(5): + b = d.getBridgesForIP(randomIPString(), "x", 1, + bridgeFilterRules=[ + filterBridgesByNotBlockedIn("cn", methodname='obfs2'), + filterBridgesByTransport('obfs2'), + ]) + try: assert len(b) > 0 + except AssertionError: + print("epic fail") + b = d.getBridgesForIP(randomIPString(), "x", 1, bridgeFilterRules=[ + filterBridgesByNotBlockedIn("us")]) + assert len(b) > 0 + + +class SQLStorageTests(unittest.TestCase): + def setUp(self): + self.fd, self.fname = tempfile.mkstemp() + self.db = bridgedb.Storage.Database(self.fname) + self.cur = self.db._conn.cursor() + + def tearDown(self): + self.db.close() + os.close(self.fd) + os.unlink(self.fname) + + def assertCloseTo(self, a, b, delta=60): + self.assertTrue(abs(a-b) <= delta) + + def testBridgeStorage(self): + db = self.db + B = bridgedb.Bridges.Bridge + t = time.time() + cur = self.cur + + k1 = "aaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbb" + k2 = "abababababababababababababababababababab" + k3 = "cccccccccccccccccccccccccccccccccccccccc" + b1 = B("serv1", "1.2.3.4", 999, fingerprint=k1) + b1_v2 = B("serv1", "1.2.3.5", 9099, fingerprint=k1) + b2 = B("serv2", "2.3.4.5", 9990, fingerprint=k2) + b3 = B("serv3", "2.3.4.6", 9008, fingerprint=k3) + validRings = ["ring1", "ring2", "ring3"] + + r = db.insertBridgeAndGetRing(b1, "ring1", t, validRings) + self.assertEquals(r, "ring1") + r = db.insertBridgeAndGetRing(b1, "ring10", t+500, validRings) + self.assertEquals(r, "ring1") + + cur.execute("SELECT distributor, address, or_port, first_seen, " + "last_seen FROM Bridges WHERE hex_key = ?", (k1,)) + v = cur.fetchone() + self.assertEquals(v, + ("ring1", "1.2.3.4", 999, + bridgedb.Storage.timeToStr(t), + bridgedb.Storage.timeToStr(t+500))) + + r = db.insertBridgeAndGetRing(b1_v2, "ring99", t+800, validRings) + self.assertEquals(r, "ring1") + cur.execute("SELECT distributor, address, or_port, first_seen, " + "last_seen FROM Bridges WHERE hex_key = ?", (k1,)) + v = cur.fetchone() + self.assertEquals(v, + ("ring1", "1.2.3.5", 9099, + bridgedb.Storage.timeToStr(t), + bridgedb.Storage.timeToStr(t+800))) + + db.insertBridgeAndGetRing(b2, "ring2", t, validRings) + db.insertBridgeAndGetRing(b3, "ring3", t, validRings) + + cur.execute("SELECT COUNT(distributor) FROM Bridges") + v = cur.fetchone() + self.assertEquals(v, (3,)) + + r = db.getEmailTime("abc(a)example.com") + self.assertEquals(r, None) + db.setEmailTime("abc(a)example.com", t) + db.setEmailTime("def(a)example.com", t+1000) + r = db.getEmailTime("abc(a)example.com") + self.assertCloseTo(r, t) + r = db.getEmailTime("def(a)example.com") + self.assertCloseTo(r, t+1000) + r = db.getEmailTime("ghi(a)example.com") + self.assertEquals(r, None) + + db.cleanEmailedBridges(t+200) + db.setEmailTime("def(a)example.com", t+5000) + r = db.getEmailTime("abc(a)example.com") + self.assertEquals(r, None) + r = db.getEmailTime("def(a)example.com") + self.assertCloseTo(r, t+5000) + cur.execute("SELECT * FROM EmailedBridges") + self.assertEquals(len(cur.fetchall()), 1) + + db.addBridgeBlock(b2.fingerprint, 'us') + self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), True) + db.delBridgeBlock(b2.fingerprint, 'us') + self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), False) + db.addBridgeBlock(b2.fingerprint, 'uk') + db.addBridgeBlock(b3.fingerprint, 'uk') + self.assertEquals(set([b2.fingerprint, b3.fingerprint]), + set(db.getBlockedBridges('uk'))) + + db.addBridgeBlock(b2.fingerprint, 'cn') + db.addBridgeBlock(b2.fingerprint, 'de') + db.addBridgeBlock(b2.fingerprint, 'jp') + db.addBridgeBlock(b2.fingerprint, 'se') + db.addBridgeBlock(b2.fingerprint, 'kr') + + self.assertEquals(set(db.getBlockingCountries(b2.fingerprint)), + set(['uk', 'cn', 'de', 'jp', 'se', 'kr'])) + self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) + db.setWarnedEmail("def(a)example.com") + self.assertEquals(db.getWarnedEmail("def(a)example.com"), True) + db.setWarnedEmail("def(a)example.com", False) + self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) + + db.setWarnedEmail("def(a)example.com") + self.assertEquals(db.getWarnedEmail("def(a)example.com"), True) + db.cleanWarnedEmails(t+200) + self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) + +class ParseDescFileTests(unittest.TestCase): + def testSimpleDesc(self): + test = "" + + for i in range(100): + test+= "".join(simpleDesc % (randomIP(), randomPort())) + test+=gettimestamp() + test+="router-signature\n" + + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + self.assertEquals(len(bs), 100) + + for b in bs: + b.assertOK() + + def testSingleOrAddress(self): + test = "" + + for i in range(100): + test+= simpleDesc % (randomIP(), randomPort()) + test+= orAddress % (randomIP(),randomPort()) + test+=gettimestamp() + test+= "router-signature\n" + + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + self.assertEquals(len(bs), 100) + + for b in bs: + b.assertOK() + + def testMultipleOrAddress(self): + test = "" + for i in range(100): + test+= simpleDesc % (randomIPString(), randomPort()) + for i in xrange(8): + test+= orAddress % (randomIPString(),randomPortSpec()) + test+=gettimestamp() + test+= "router-signature\n" + + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + self.assertEquals(len(bs), 100) + + for b in bs: + b.assertOK() + + def testConvolutedOrAddress(self): + test = "" + for i in range(100): + test+= simpleDesc % (randomIPString(), randomPort()) + for i in xrange(8): + test+= orAddress % (randomIPString(),randomPortSpec()) + test+=gettimestamp() + test+= "router-signature\n" + + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + self.assertEquals(len(bs), 100) + + for b in bs: + b.assertOK() + + def testParseCountryBlockFile(self): + simpleBlock = "%s:%s %s\n" + countries = ['us', 'nl', 'de', 'cz', 'sk', 'as', 'si', 'it'] + test = str() + for i in range(100): + test += simpleBlock % (randomIPString(), randomPort(), + randomCountrySpec()) + test+=gettimestamp() + + for a,p,c in bridgedb.Bridges.parseCountryBlockFile(test.split('\n')): + assert type(a) in (ipaddr.IPv6Address, ipaddr.IPv4Address) + assert isinstance(p, addr.PortList) + assert isinstance(c, list) + assert len(c) > 0 + for y in c: + assert y in countries + #print "address: %s" % a + #print "portlist: %s" % p + #print "countries: %s" % c + +class BridgeStabilityTests(unittest.TestCase): + def setUp(self): + self.fd, self.fname = tempfile.mkstemp() + self.db = bridgedb.Storage.Database(self.fname) + bridgedb.Storage.setDB(self.db) + self.cur = self.db._conn.cursor() + + def tearDown(self): + self.db.close() + os.close(self.fd) + os.unlink(self.fname) + + def testAddOrUpdateSingleBridgeHistory(self): + db = self.db + b = fakeBridge() + timestamp = time.time() + bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b, timestamp) + assert isinstance(bhe, BridgeHistory) + assert isinstance(db.getBridgeHistory(b.fingerprint), BridgeHistory) + assert len([y for y in db.getAllBridgeHistory()]) == 1 + + def testDeletingSingleBridgeHistory(self): + db = self.db + b = fakeBridge() + timestamp = time.time() + bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b, timestamp) + assert isinstance(bhe, BridgeHistory) + assert isinstance(db.getBridgeHistory(b.fingerprint), BridgeHistory) + db.delBridgeHistory(b.fingerprint) + assert db.getBridgeHistory(b.fingerprint) is None + assert len([y for y in db.getAllBridgeHistory()]) == 0 + + def testTOSA(self): + db = self.db + b = random.choice([fakeBridge,fakeBridge6])() + def timestampSeries(x): + for i in xrange(61): + yield (i+1)*60*30 + x # 30 minute intervals + now = time.time() + time_on_address = long(60*30*60) # 30 hours + downtime = 60*60*random.randint(0,4) # random hours of downtime + + for t in timestampSeries(now): + bridgedb.Stability.addOrUpdateBridgeHistory(b,t) + assert db.getBridgeHistory(b.fingerprint).tosa == time_on_address + + b.orport += 1 + + for t in timestampSeries(now + time_on_address + downtime): + bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b,t) + assert db.getBridgeHistory(b.fingerprint).tosa == time_on_address + downtime + + def testLastSeenWithDifferentAddressAndPort(self): + db = self.db + for i in xrange(10): + num_desc = 30 + time_start = time.time() + ts = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] + b = random.choice([fakeBridge(), fakeBridge6()]) + [ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ] + + # change the port + b.orport = b.orport+1 + last_seen = ts[-1] + ts = [ 60*30*(i+1) + last_seen for i in xrange(num_desc) ] + [ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ] + b = db.getBridgeHistory(b.fingerprint) + assert b.tosa == ts[-1] - last_seen + assert (long(last_seen*1000) == b.lastSeenWithDifferentAddressAndPort) + assert (long(ts[-1]*1000) == b.lastSeenWithThisAddressAndPort) + + def testFamiliar(self): + # create some bridges + # XXX: slow + num_bridges = 10 + num_desc = 4*48 # 30m intervals, 48 per day + time_start = time.time() + bridges = [ fakeBridge() for x in xrange(num_bridges) ] + t = time.time() + ts = [ (i+1)*60*30+t for i in xrange(num_bridges) ] + for b in bridges: + time_series = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] + [ bridgedb.Stability.addOrUpdateBridgeHistory(b, i) for i in time_series ] + assert None not in bridges + # +1 to avoid rounding errors + assert bridges[-(num_bridges/8 + 1)].familiar == True + + def testDiscountAndPruneBridgeHistory(self): + """ Test pruning of old Bridge History """ + if os.environ.get('TRAVIS'): + self.skipTest("Hangs on Travis-CI.") + + db = self.db + + # make a bunch of bridges + num_bridges = 20 + time_start = time.time() + bridges = [random.choice([fakeBridge, fakeBridge6])() + for i in xrange(num_bridges)] + + # run some of the bridges for the full time series + running = bridges[:num_bridges/2] + # and some that are not + expired = bridges[num_bridges/2:] + + for b in running: assert b not in expired + + # Solving: + # 1 discount event per 12 hours, 24 descriptors 30m apart + num_successful = random.randint(2,60) + # figure out how many intervals it will take for weightedUptime to + # decay to < 1 + num_desc = int(30*log(1/float(num_successful*30*60))/(-0.05)) + timeseries = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] + + for i in timeseries: + for b in running: + bridgedb.Stability.addOrUpdateBridgeHistory(b, i) + + if num_successful > 0: + for b in expired: + bridgedb.Stability.addOrUpdateBridgeHistory(b, i) + num_successful -= 1 + + # now we expect to see the bridge has been removed from history + for bridge in expired: + b = db.getBridgeHistory(bridge.fingerprint) + assert b is None + # and make sure none of the others have + for bridge in running: + b = db.getBridgeHistory(bridge.fingerprint) + assert b is not None + +def testSuite(): + suite = unittest.TestSuite() + loader = unittest.TestLoader() + + for klass in [IPBridgeDistTests, SQLStorageTests, EmailBridgeDistTests, + ParseDescFileTests, BridgeStabilityTests]: + suite.addTest(loader.loadTestsFromTestCase(klass)) + + for module in [ bridgedb.Bridges, + bridgedb.Main, + bridgedb.Dist, + bridgedb.schedule ]: + suite.addTest(doctest.DocTestSuite(module)) + + return suite + +def main(): + suppressWarnings() + + unittest.TextTestRunner(verbosity=1).run(testSuite()) + diff --git a/lib/bridgedb/test/test_Tests.py b/lib/bridgedb/test/test_Tests.py index e57f4a1..53d0e81 100644 --- a/lib/bridgedb/test/test_Tests.py +++ b/lib/bridgedb/test/test_Tests.py @@ -9,8 +9,9 @@ # (c) 2007-2013, all entities within the AUTHORS file # :license: 3-Clause BSD, see LICENSE for licensing information -"""Class wrappers to adapt BridgeDB old unittests in :mod:`bridgedb.Tests` to -be compatible with the newer :api:`twisted.trial` unittests in this directory. +"""Class wrappers to adapt BridgeDB old unittests in :mod:`bridgedb.Tests` +(now kept in :mod:`bridgedb.test.legacy_Tests`) to be compatible with the +newer :api:`twisted.trial` unittests in this directory. """ from __future__ import print_function @@ -26,11 +27,11 @@ import warnings from twisted.python import monkey from twisted.trial import unittest -from bridgedb import Tests +from bridgedb.test import legacy_Tests as Tests from bridgedb.test import deprecated -warnings.filterwarnings('ignore', module="bridgedb\.Tests") +warnings.filterwarnings('ignore', module="bridgedb\.test\.legacy_Tests") pyunit = __import__('unittest')

1 0

[bridgedb/master] Change BridgeRequestBase.isValid() to allow setting the attribute.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 1cc15e8c0844c704f17502c063b2caf384977e06 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 23:50:11 2014 +0000 Change BridgeRequestBase.isValid() to allow setting the attribute. The syntax is nicer with: >>> bridgeRequest = bridgedb.bridgerequest.BridgeRequestBase() >>> bridgeRequest.isValid() False >>> bridgeRequest.isValid(True) >>> bridgeRequest.isValid() True than it was with the old implementation: >>> bridgeRequest = bridgedb.bridgerequest.BridgeRequestBase() >>> bridgeRequest.isValid() False >>> bridgeRequest.valid = True >>> bridgeRequest.isValid() True --- lib/bridgedb/bridgerequest.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index 7b2bfe9..3ef0dc7 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -82,8 +82,18 @@ class BridgeRequestBase(object): self.notBlockedIn = list() self.valid = False - def isValid(self): - pass + def isValid(self, valid=None): + """Set or determine if this request was valid. + + :type valid: None or bool + :param valid: If ``None``, get the current request validity. If + ``True`` or ``False``, set the request validity accordingly. + :rtype: bool + :returns: Whether or not this request is valid. + """ + if isinstance(valid, bool): + self.valid = valid + return self.valid def withIPv4(self): self.addressClass = ipaddr.IPv4Address

1 0

[bridgedb/master] Add `client` attribute and `getHashringPlacement()` to IRequestBridges.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit ef1e219f715e09d525561b0a90cfbf00df45d9f5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 23:56:11 2014 +0000 Add `client` attribute and `getHashringPlacement()` to IRequestBridges. * ADD interface definitions for IRequestBridges.client and IRequestBridges.getHashringPlacement(). --- lib/bridgedb/bridgerequest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index 3ef0dc7..840b2fc 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -36,6 +36,12 @@ class IRequestBridges(Interface): "not be blocked in these countries.") valid = Attribute( "A boolean. Should be ``True`` if the client's request was valid.") + client = Attribute( + "This should be some information unique to the client making the " + "request for bridges, such that we are able to HMAC this unique " + "data, via getHashringPlacement(), in order to place the client " + "into a hashring (determining which bridge addresses they get in " + "the request response).") def addFilter(): """Add a filter to the list of ``filters``.""" @@ -49,6 +55,13 @@ class IRequestBridges(Interface): ``addressClass``. """ + def getHashringPlacement(): + """Use some unique parameters of the client making this request to + obtain a value which we can use to place them into one of the hashrings + with :class:`~bridgedb.bridges.Bridge`s in it, in order to give that + client different bridges than other clients. + """ + def isValid(): """Determine if the request is ``valid`` according to some parameters."""

1 0

[bridgedb/master] Implement `client` and `getHashringPlacement()` interfaces.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit eb28b43d69ea3d91b8ccab6fbc7569817fc25ebb Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 23:57:36 2014 +0000 Implement `client` and `getHashringPlacement()` interfaces. * ADD base class implementations of IRequestBridges.client and IRequestBridges.getHashringPlacement() to BridgeRequestBase class. --- lib/bridgedb/bridgerequest.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index 840b2fc..d7b4d18 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -20,6 +20,7 @@ from zope.interface import Attribute from zope.interface import Interface from bridgedb import Filters +from bridgedb.crypto import getHMACFunc class IRequestBridges(Interface): @@ -93,8 +94,34 @@ class BridgeRequestBase(object): self.filters = list() self.transports = list() self.notBlockedIn = list() + #: This should be some information unique to the client making the + #: request for bridges, such that we are able to HMAC this unique data + #: in order to place the client into a hashring (determining which + #: bridge addresses they get in the request response). It defaults to + #: the string ``'default'``. + self.client = 'default' self.valid = False + def getHashringPlacement(self, key, client=None): + """Create an HMAC of some **client** info using a **key**. + + :param str key: The key to use for HMACing. + :param str client: Some (hopefully unique) information about the + client who is requesting bridges, such as an IP or email address. + :rtype: long + :returns: A long specifying index of the first node in a hashring to + be distributed to the client. This value should obviously be used + mod the number of nodes in the hashring. + """ + if not client: + client = self.client + + # Get an HMAC with the key of the client identifier: + digest = getHMACFunc(key)(client) + # Take the lower 8 bytes of the digest and convert to a long: + position = long(digest[:8], 16) + return position + def isValid(self, valid=None): """Set or determine if this request was valid.

1 0

[bridgedb/master] Implement bridgedb.bridges.Flags for storing/parsing flags.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 62f6039b435058c711a6918fd153f85d9889012b Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Dec 6 00:04:52 2014 +0000 Implement bridgedb.bridges.Flags for storing/parsing flags. * ADD bridgedb.bridges.Flags class. * CHANGE bridgedb.bridges.Bridge.flags attribute to be an instance of bridgedb.bridges.Flags. --- lib/bridgedb/bridges.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 1ab88f1..21d9257 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -42,6 +42,53 @@ class ServerDescriptorWithoutNetworkstatus(MalformedBridgeInfo): mentioned in the latest ``@type bridge-networkstatus`` document. """ +class Flags(object): + """All the flags which a :class:`Bridge` may have.""" + + fast = False + guard = False + running = False + stable = False + valid = False + + def update(self, flags): + """Update with **flags** taken from an ``@type networkstatus-bridge`` + 's'-line. + + From `dir-spec.txt`_: + | + | "s" SP Flags NL + | + | [Exactly once.] + | + | A series of space-separated status flags, in lexical order (as ASCII + | byte strings). Currently documented flags are: + | + | [...] + | "Fast" if the router is suitable for high-bandwidth circuits. + | "Guard" if the router is suitable for use as an entry guard. + | [...] + | "Stable" if the router is suitable for long-lived circuits. + | "Running" if the router is currently usable. + | [...] + | "Valid" if the router has been 'validated'. + + .. _dir-spec.txt: + https://gitweb.torproject.org/torspec.git/blob/7647f6d4d:/dir-spec.txt#l1603 + + :param list flags: A list of strings containing each of the flags + parsed from the 's'-line. + """ + self.fast = 'Fast' in flags + self.guard = 'Guard' in flags + self.running = 'Running' in flags + self.stable = 'Stable' in flags + self.valid = 'Valid' in flags + + if self.fast or self.guard or self.running or self.stable or self.valid: + logging.debug("Parsed Flags: %s" % ' '.join(flags)) + + class PluggableTransport(object): """A single instance of a Pluggable Transport (PT) offered by a :class:`Bridge`. @@ -357,7 +404,7 @@ class Bridge(object): self.dirPort = 0 # ``DirPort`` set to ``0`` self.orAddresses = [] self.transports = [] - self.flags = BridgeFlags() + self.flags = Flags() self.hibernating = False self.bandwidth = None

1 0

[bridgedb/master] Add methods to bridgedb.bridges.Bridge for backwards compatibility.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 0c1f4edccabb752266b628e87090f3d72f17f6be Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Dec 6 00:56:20 2014 +0000 Add methods to bridgedb.bridges.Bridge for backwards compatibility. * ADD several methods and attributes for backwards compatibility, contained within the Bridge._backwardsCompatible() method, the latter is only activated if the Bridge is initialised with *args and **kwargs (as was the case with the old Bridge class). The methods and attributes implementing backwards compatibility are not available if the Bridge class is initialised in the new way, i.e. via descriptors parsed with Stem. --- lib/bridgedb/bridges.py | 161 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 159 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index ecbf863..100dd12 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -9,13 +9,22 @@ """Classes for manipulating and storing Bridges and their attributes.""" +from __future__ import print_function import ipaddr import logging import os +from bridgedb import safelog +from bridgedb import bridgerequest +from bridgedb.parse.addr import isIPAddress +from bridgedb.parse.addr import isIPv6 from bridgedb.parse.addr import isValidIP +from bridgedb.parse.addr import PortList from bridgedb.parse.fingerprint import isValidFingerprint +from bridgedb.parse.fingerprint import toHex +from bridgedb.parse.fingerprint import fromHex + class MalformedBridgeInfo(ValueError): @@ -394,8 +403,21 @@ class Bridge(object): #: created with the ``signing-key`` contained in that descriptor. _checkServerDescriptorSignature = True - def __init__(self): - """Create a new ``Bridge``.""" + def __init__(self, + # for backwards compatibility: + nickname=None, ip=None, orport=None, + fingerprint=None, id_digest=None, or_addresses=None): + """Create a and store information for a new ``Bridge``. + + .. info: For backwards compatibility, `nickname`, `ip`, and `orport` + must be the first, second, and third arguments, respectively. The + `fingerprint` and `id_digest` were previously kwargs, and are also + provided for backwards compatibility. New calls to + :meth:`__init__` *should* avoid using these kwargs, and instead + use the methods :meth:`updateFromNetworkStatus`, + :meth:`updateFromServerDescriptor`, and + :meth:`updateFromExtraInfoDescriptor`. + """ self.fingerprint = None self.nickname = None self.address = None @@ -435,6 +457,141 @@ class Bridge(object): self.descriptorDigest = None self.extrainfoDigest = None + # For backwards compatibility with the old, deprecated version of this + # class: + if nickname or ip or orport or fingerprint or id_digest: + self._backwardsCompatible(nickname=nickname, address=ip, + orPort=orport, fingerprint=fingerprint, + idDigest=id_digest, + orAddresses=or_addresses) + + def _backwardsCompatible(self, nickname=None, address=None, orPort=None, + fingerprint=None, idDigest=None, + orAddresses=None): + """Functionality for maintaining backwards compatibility with the older + version of this class (see :class:`bridgedb.test.deprecated.Bridge`). + """ + def getID(): + """Get the binary encoded form of this ``Bridge``'s ``fingerprint``. + + This method is provided for backwards compatibility and should not + be relied upon. + """ + if self.fingerprint: + return fromHex(self.fingerprint) + self.getID = getID + + def setDescriptorDigest(digest): + """Set this ``Bridge``'s server-descriptor digest. + + This method is provided for backwards compatibility and should not + be relied upon. + """ + self.desc_digest = digest # old attribute for backwards compat + self.descriptorDigest = digest # new attribute + self.desc_digest = None + self.setDescriptorDigest = setDescriptorDigest + + def setExtraInfoDigest(digest): + """Set this ``Bridge``'s extrainfo digest. + + This method is provided for backwards compatibility and should not + be relied upon. + """ + self.ei_digest = digest # old attribute for backwards compat + self.extrainfoDigest = digest # new attribute + self.ei_digest = None + self.setExtraInfoDigest = setExtraInfoDigest + + def setStatus(running=None, stable=None): + """Set this ``Bridge``'s "Running" and "Stable" flags. + + This method is provided for backwards compatibility and should not + be relied upon. + """ + if running is not None: + self.running = bool(running) + self.flags.running = bool(running) + if stable is not None: + self.stable = bool(stable) + self.flags.stable = bool(running) + self.running = False + self.stable = False + self.setStatus = setStatus + + def getConfigLine(includeFingerprint=False, addressClass=None, + request=None, transport=None): + """Get a vanilla bridge line for this ``Bridge``. + + This method is provided for backwards compatibility and should not + be relied upon. + + The old ``bridgedb.Bridges.Bridge.getConfigLine()`` method didn't + know about :class:`~bridgedb.bridgerequest.BridgeRequestBase`s, + and so this modified version is backwards compatible by creating a + :class:`~bridgedb.bridgerequest.BridgeRequestBase` for + :meth:`getBridgeLine`. The default parameters are the same as they + were in the old ``bridgedb.Bridges.Bridge`` class. + + :param bool includeFingerprint: If ``True``, include the + ``fingerprint`` of this :class:`Bridge` in the returned bridge + line. + :type addressClass: :class:`ipaddr.IPv4Address` or + :class:`ipaddr.IPv6Address`. + :param addressClass: Type of address to choose. + :param str request: A string unique to this request + e.g. email-address or ``uniformMap(ip)`` or ``'default'``. In + this case, this is not a + :class:`~bridgerequest.BridgeRequestBase` (as might be + expected) but the equivalent of + :data:`bridgerequest.BridgeRequestBase.client`. + :param str transport: A pluggable transport method name. + """ + bridgeRequest = bridgerequest.BridgeRequestBase(addressClass) + bridgeRequest.client = request if request else bridgeRequest.client + bridgeRequest.isValid(True) + + if transport: + bridgeRequest.withPluggableTransportType(transport) + + bridgeRequest.generateFilters() + bridgeLine = self.getBridgeLine(bridgeRequest, includeFingerprint) + return bridgeLine + self.getConfigLine = getConfigLine + + if nickname: + self.nickname = nickname # XXX check nickname spec conformity? + if address: + self.address = address # XXX check validip? + if orPort: + self.orPort = orPort # XXX check validity? + + if idDigest: + if not fingerprint: + if not len(idDigest) == 20: + raise TypeError("Bridge with invalid ID") + self.fingerprint = toHex(idDigest) + elif fingerprint: + if not isValidFingerprint(fingerprint): + raise TypeError("Bridge with invalid fingerprint (%r)" + % fingerprint) + self.fingerprint = fingerprint.lower() + else: + raise TypeError("Bridge with no ID") + + if orAddresses and isinstance(orAddresses, dict): + for ip, portlist in orAddresses.items(): + validAddress = isIPAddress(ip, compressed=False) + if validAddress: + # The old code expected a `bridgedb.parse.addr.PortList`: + if isinstance(portlist, PortList): + for port in portlist.ports: + self.orAddresses.append( + (validAddress, port, validAddress.version,)) + else: + self.orAddresses.append( + (validAddress, port, validAddress.version)) + def _checkServerDescriptor(self, descriptor): # If we're parsing the server-descriptor, require a networkstatus # document:

1 0

[bridgedb/master] Add method for verifying a Bridge's extrainfo router-signature.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 6b4c3f08545009a68cb6b2a24c2357603e484609 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 5 18:43:49 2014 -0800 Add method for verifying a Bridge's extrainfo router-signature. * ADD new method bridgedb.bridges.Bridge._verifyExtraInfoSignature(). * ADD new function bridgedb.crypto.removePKCS1Padding(). --- lib/bridgedb/bridges.py | 109 ++++++++++++++++++++++++++++++++++++++++++++++- lib/bridgedb/crypto.py | 41 ++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 17486e2..d2e528d 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -11,13 +11,20 @@ from __future__ import print_function +import base64 +import codecs import hashlib import ipaddr import logging import os +from Crypto.Util import asn1 +from Crypto.Util.number import bytes_to_long +from Crypto.Util.number import long_to_bytes + from bridgedb import safelog from bridgedb import bridgerequest +from bridgedb.crypto import removePKCS1Padding from bridgedb.parse.addr import isIPAddress from bridgedb.parse.addr import isIPv6 from bridgedb.parse.addr import isValidIP @@ -55,6 +62,10 @@ class ServerDescriptorWithoutNetworkstatus(MalformedBridgeInfo): mentioned in the latest ``@type bridge-networkstatus`` document. """ +class InvalidExtraInfoSignature(MalformedBridgeInfo): + """Raised if the signature on an ``@type bridge-extrainfo`` is invalid.""" + + class Flags(object): """All the flags which a :class:`Bridge` may have.""" @@ -1174,7 +1185,91 @@ class Bridge(object): self.extrainfoDigest = descriptor.extrainfoDigest - def updateFromExtraInfoDescriptor(self, descriptor): + def _verifyExtraInfoSignature(self, descriptor): + """Verify the signature on the contents of this :class:`Bridge`'s + ``@type bridge-extrainfo`` descriptor. + + :type descriptor: + :api:`stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor` + :param descriptor: An ``@type bridge-extrainfo`` descriptor for this + :class:`Bridge`, parsed with Stem. + :raises InvalidExtraInfoSignature: if the signature was invalid, + missing, malformed, or couldn't be verified successfully. + :returns: ``None`` if the signature was valid and verifiable. + """ + # The blocksize is always 128 bits for a 1024-bit key + BLOCKSIZE = 128 + + TOR_SIGNING_KEY_HEADER = u'-----BEGIN RSA PUBLIC KEY-----\n' + TOR_SIGNING_KEY_FOOTER = u'-----END RSA PUBLIC KEY-----' + TOR_BEGIN_SIGNATURE = u'-----BEGIN SIGNATURE-----\n' + TOR_END_SIGNATURE = u'-----END SIGNATURE-----\n' + + logging.info("Verifying extrainfo signature for %s..." % self) + + # Get the bytes of the descriptor signature without the headers: + document, signature = descriptor.get_bytes().split(TOR_BEGIN_SIGNATURE) + signature = signature.replace(TOR_END_SIGNATURE, '') + signature = signature.replace('\n', '') + signature = signature.strip() + + try: + # Get the ASN.1 sequence: + sequence = asn1.DerSequence() + + key = self.signingKey + key = key.strip(TOR_SIGNING_KEY_HEADER) + key = key.strip(TOR_SIGNING_KEY_FOOTER) + key = key.replace('\n', '') + key = base64.b64decode(key) + + sequence.decode(key) + + modulus = sequence[0] + publicExponent = sequence[1] + + # The public exponent of RSA signing-keys should always be 65537, + # but we're not going to turn them down if they want to use a + # potentially dangerous exponent. + if publicExponent != 65537: # pragma: no cover + logging.warn("Odd RSA exponent in signing-key for %s: %s" % + (self, publicExponent)) + + # Base64 decode the signature: + signatureDecoded = base64.b64decode(signature) + + # Convert the signature to a long: + signatureLong = bytes_to_long(signatureDecoded) + + # Decrypt the long signature with the modulus and public exponent: + decryptedInt = pow(signatureLong, publicExponent, modulus) + + # Then convert it back to a byte array: + decryptedBytes = long_to_bytes(decryptedInt, BLOCKSIZE) + + # Remove the PKCS#1 padding from the signature: + unpadded = removePKCS1Padding(decryptedBytes) + + # This is the hexadecimal SHA-1 hash digest of the descriptor document + # as it was signed: + signedDigest = codecs.encode(unpadded, 'hex_codec') + actualDigest = hashlib.sha1(document).hexdigest() + + except Exception as error: + logging.debug("Error verifying extrainfo signature: %s" % error) + raise InvalidExtraInfoSignature( + "Extrainfo signature for %s couldn't be decoded: %s" % + (self, signature)) + else: + if signedDigest != actualDigest: + raise InvalidExtraInfoSignature( + ("The extrainfo digest signed by bridge %s didn't match the " + "actual digest.\nSigned digest: %s\nActual digest: %s") % + (self, signedDigest, actualDigest)) + else: + logging.info("Extrainfo signature was verified successfully!") + + def updateFromExtraInfoDescriptor(self, descriptor, verify=True): """Update this bridge's information from an extrainfo descriptor. .. todo:: The ``transport`` attribute of Stem's @@ -1186,5 +1281,17 @@ class Bridge(object): :type descriptor: :api:`stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor` :param descriptor: DOCDOC + :param bool verify: If ``True``, check that the ``router-signature`` + on the extrainfo **descriptor** is a valid signature from + :data:`signingkey`. """ + if verify: + try: + self._verifyExtraInfoSignature(descriptor) + except InvalidExtraInfoSignature as error: + logging.warn(error) + logging.info(("Tossing extrainfo descriptor due to an invalid " + "signature.")) + return + self.descriptors['extrainfo'] = descriptor diff --git a/lib/bridgedb/crypto.py b/lib/bridgedb/crypto.py index e8cf8fd..63e532b 100644 --- a/lib/bridgedb/crypto.py +++ b/lib/bridgedb/crypto.py @@ -87,6 +87,9 @@ GPGME_CONTEXT_HOMEDIR = '.gnupg' GPGME_CONTEXT_BINARY = which('gpg2') or which('gpg') # These will be lists +class PKCS1PaddingError(Exception): + """Raised when there is a problem adding or removing PKCS#1 padding.""" + class RSAKeyGenerationError(Exception): """Raised when there was an error creating an RSA keypair.""" @@ -273,6 +276,44 @@ def getHMACFunc(key, hex=True): return h_tmp.digest() return hmac_fn +def removePKCS1Padding(message): + """Remove PKCS#1 padding from a **message**. + + (PKCS#1 v1.0? see https://bugs.torproject.org/13042) + + Each block is 128 bytes total in size: + + * 2 bytes for the type info ('\x00\x01') + * 1 byte for the separator ('\x00') + * variable length padding ('\xFF') + * variable length for the **message** + + For more information on the structure of PKCS#1 padding, see :rfc:`2313`, + particularly the notes in §8.1. + + :param str message: A message which is PKCS#1 padded. + :raises PKCS1PaddingError: if there is an issue parsing the **message**. + :rtype: bytes + :returns: The message without the PKCS#1 padding. + """ + padding = b'\xFF' + typeinfo = b'\x00\x01' + separator = b'\x00' + + unpadded = None + + try: + if message.index(typeinfo) != 0: + raise PKCS1PaddingError("Couldn't find PKCS#1 identifier bytes!") + start = message.index(separator, 2) + 1 # 2 bytes for the typeinfo, + # and 1 byte for the separator. + except ValueError: + raise PKCS1PaddingError("Couldn't find PKCS#1 separator byte!") + else: + unpadded = message[start:] + + return unpadded + def _createGPGMEErrorInterpreters(): """Create a mapping of GPGME ERRNOs ←→ human-readable error names/causes.

1 0

[bridgedb/master] Catch ValueErrors while coercing PluggableTransport port numbers.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 83378640b09dc8a6c6d5a58389fd36c25dab37b1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Dec 6 00:17:24 2014 +0000 Catch ValueErrors while coercing PluggableTransport port numbers. --- lib/bridgedb/bridges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 21d9257..ecbf863 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -240,7 +240,7 @@ class PluggableTransport(object): try: # Coerce the port to be an integer: self.port = int(self.port) - except TypeError: + except (TypeError, ValueError): raise MalformedPluggableTransport( ("Cannot create PluggableTransport with port type: %s.") % type(self.port))

1 0