commit 5e032efa39a4c00f00ca2f86460f952bf2c081a8 Author: Isis Lovecruft <isis@torproject.org> Date: Wed Apr 1 01:04:33 2015 +0000 Remove all remnants of #5027/#5418 bridge block parser code. * REMOVES bridgedb.Bridges.re_ipv4. * REMOVES bridgedb.Bridges.re_ipv6. * REMOVES bridgedb.Bridges.parseCountryBlockFile(). * REMOVES bridgedb.Bridges.BridgeBlock. * REMOVES bridgedb.Bridges.CountryBlock. * REMOVES bridgedb.Storage.Database.addBridgeBlock(). * REMOVES bridgedb.Storage.Database.delBridgeBlock(). * REMOVES bridgedb.Storage.Database.cleanBridgeBlocks(). * REMOVES bridgedb.Storage.Database.getBlockingCountries(). * REMOVES bridgedb.Storage.Database.getBlockedBridges(). * REMOVES bridgedb.Storage.Database.isBlocked(). * REMOVES several asserts for the above b.Storage.Database methods from bridgedb.test.legacy_Tests.SQLStorageTests. * REMOVES bridgedb.test.legacy_Tests.ParseCountryBlockFileTests. * FIXES #5418: https://bugs.torproject.org/5418 * SEEALSO #5027: https://bugs.torproject.org/5027 * FIXES part of #12505: https://bugs.torproject.org/12505 --- lib/bridgedb/Bridges.py | 72 ------------------------------------- lib/bridgedb/Bucket.py | 8 ----- lib/bridgedb/Main.py | 20 ----------- lib/bridgedb/Storage.py | 46 ------------------------ lib/bridgedb/test/legacy_Tests.py | 40 +-------------------- 5 files changed, 1 insertion(+), 185 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 078cfff..0239730 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -40,41 +40,6 @@ ID_LEN = 20 # XXX Only used in commented out line in Storage.py DIGEST_LEN = 20 PORTSPEC_LEN = 16 -re_ipv6 = re.compile("\[([a-fA-F0-9:]+)\]:(.*$)") -re_ipv4 = re.compile("((?:\d{1,3}\.?){4}):(.*$)") - - -def parseCountryBlockFile(f): - """Generator. Parses a blocked-bridges file 'f', and yields - a fingerprint (ID), address, a list of ports, and a list of country - codes where the bridge is blocked for each valid line: - address, port [], countrycode []""" - for line in f: - ID = address = fields = portlist = countries = None - line = line.strip() - try: - ID, addrspec, countries = line.split() - if isValidFingerprint(ID): - ID = fromHex(ID) - logging.debug("Parsed ID: %s", ID) - else: - print "failed to parse ID!" - continue # skip this line - - for regex in [re_ipv4, re_ipv6]: - m = regex.match(addrspec) - if m: - address = ipaddr.IPAddress(m.group(1)) - portlist = addr.PortList(m.group(2)) - countries = countries.split(',') - logging.debug("Parsed address: %s", address) - logging.debug("Parsed portlist: %s", portlist) - logging.debug("Parsed countries: %s", countries) - except (IndexError, ValueError): - logging.debug("Skipping line") - continue # skip this line - if ID and address and portlist and countries: - yield ID, address, portlist, countries class BridgeHolder(object): """Abstract base class for all classes that hold bridges.""" @@ -751,40 +716,3 @@ class FilteredBridgeSplitter(BridgeHolder): desc = "%s %s" % (description.strip(), " ".join([v for k,v in grouped.items()]).strip()) f.write("%s %s\n"%( toHex(b.getID()), desc)) - -class BridgeBlock(object): - """Base class that abstracts bridge blocking. - - .. TODO:: This should be a zope.interface specification. - """ - def __init__(self): - pass - - def insert(self, fingerprint, blockingRule): - raise NotImplementedError - - def clear(self): - pass - -class CountryBlock(BridgeBlock): - """Countrywide bridge blocking""" - def __init__(self): - self.db = bridgedb.Storage.getDB() - - def clear(self): - assert self.db - self.db.cleanBridgeBlocks() - self.db.commit() - - def insert(self, fingerprint, blockingRule): - """ insert a country based blocking rule """ - assert self.db - countryCode = blockingRule - self.db.addBridgeBlock(fingerprint, countryCode) - self.db.commit() - - def getBlockingCountries(self, fingerprint): - """ returns a list of country codes where this fingerprint is blocked""" - assert self.db - if fingerprint is not None: - return self.db.getBlockingCountries(fingerprint) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index 5e7f61b..1382188 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -242,14 +242,6 @@ class BucketManager(object): bridgeHistories.sort(lambda x,y: cmp(x.weightedFractionalUptime, y.weightedFractionalUptime)) - # for a bridge, get the list of countries it might not work in - blocklist = dict() - if getattr(self.cfg, "COUNTRY_BLOCK_FILE", None) is not None: - f = open(self.cfg.COUNTRY_BLOCK_FILE, 'r') - for ID,address,portlist,countries in bridgedb.Bridges.parseCountryBlockFile(f): - blocklist[toHex(ID)] = countries - f.close() - try: f = open(filename, 'w') if self.cfg.COLLECT_TIMESTAMPS: diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 3b018f9..d5f4c24 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -175,26 +175,6 @@ def load(state, splitter, clear=False): "but could not find bridge with that fingerprint.") % router.fingerprint) - # XXX TODO refactor the next block according with new parsers for OONI - # bridge-reachability reports: - if state.COUNTRY_BLOCK_FILE: # pragma: no cover - logging.info("Opening Blocking Countries file %s" - % state.COUNTRY_BLOCK_FILE) - f = open(state.COUNTRY_BLOCK_FILE) - # Identity digest, primary OR address, portlist, country codes - for ID, addr, portlist, cc in Bridges.parseCountryBlockFile(f): - if ID in bridges.keys() and bridges[ID].running: - for port in portlist: - addrport = "{0}:{1}".format(addr, port) - logging.debug(":'( Tears! %s blocked bridge %s at %s" - % (cc, bridges[ID].fingerprint, addrport)) - try: - bridges[ID].blockingCountries[addrport].update(cc) - except KeyError: - bridges[ID].blockingCountries[addrport] = set(cc) - logging.debug("Closing blocking-countries document") - f.close() - inserted = 0 logging.info("Inserting %d bridges into splitter..." % len(bridges)) for fingerprint, bridge in bridges.items(): diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 1efb139..439634f 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -243,52 +243,6 @@ class Database(object): cur.execute("UPDATE Bridges SET distributor = ? WHERE hex_key = ?", (distributor, hex_key)) - def addBridgeBlock(self, fingerprint, countryCode): - cur = self._cur - cur.execute("INSERT OR REPLACE INTO BlockedBridges " - "(hex_key,blocking_country) VALUES (?,?)", - (fingerprint, countryCode)) - - def delBridgeBlock(self, fingerprint, countryCode): - cur = self._cur - cur.execute("DELETE FROM BlockedBridges WHERE hex_key = ? " - "AND blocking_country = ?", (fingerprint, countryCode)) - - def cleanBridgeBlocks(self): - cur = self._cur - cur.execute("DELETE FROM BlockedBridges") - - def getBlockingCountries(self, fingerprint): - cur = self._cur - cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE hex_key = ? ", - (fingerprint,)) - v = cur.fetchall() - if v is None: - return None - - # return list of country-codes - return [ str(result[1]) for (result) in v ] - - def getBlockedBridges(self, countryCode): - cur = self._cur - cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE blocking_country = ? ", - (countryCode,)) - v = cur.fetchall() - if v is None: - return None - # return list of fingerprints - return [ str(result[0]) for (result) in v ] - - def isBlocked(self, fingerprint, countryCode): - cur = self._cur - cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE " - "hex_key = ? AND blocking_country = ?", - (fingerprint, countryCode)) - v = cur.fetchone() - if v is None: - return False - return True - def getWarnedEmail(self, addr): addr = hashlib.sha1(addr).hexdigest() cur = self._cur diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index cd0c6de..4d0c53c 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -501,23 +501,6 @@ class SQLStorageTests(unittest.TestCase): cur.execute("SELECT * FROM EmailedBridges") self.assertEquals(len(cur.fetchall()), 1) - db.addBridgeBlock(b2.fingerprint, 'us') - self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), True) - db.delBridgeBlock(b2.fingerprint, 'us') - self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), False) - db.addBridgeBlock(b2.fingerprint, 'uk') - db.addBridgeBlock(b3.fingerprint, 'uk') - self.assertEquals(set([b2.fingerprint, b3.fingerprint]), - set(db.getBlockedBridges('uk'))) - - db.addBridgeBlock(b2.fingerprint, 'cn') - db.addBridgeBlock(b2.fingerprint, 'de') - db.addBridgeBlock(b2.fingerprint, 'jp') - db.addBridgeBlock(b2.fingerprint, 'se') - db.addBridgeBlock(b2.fingerprint, 'kr') - - self.assertEquals(set(db.getBlockingCountries(b2.fingerprint)), - set(['uk', 'cn', 'de', 'jp', 'se', 'kr'])) self.assertEquals(db.getWarnedEmail("def@example.com"), False) db.setWarnedEmail("def@example.com") self.assertEquals(db.getWarnedEmail("def@example.com"), True) @@ -529,27 +512,6 @@ class SQLStorageTests(unittest.TestCase): db.cleanWarnedEmails(t+200) self.assertEquals(db.getWarnedEmail("def@example.com"), False) -class ParseCountryBlockFileTests(unittest.TestCase): - - def testParseCountryBlockFile(self): - simpleBlock = "%s:%s %s\n" - countries = ['us', 'nl', 'de', 'cz', 'sk', 'as', 'si', 'it'] - test = str() - for i in range(100): - test += simpleBlock % (randomIPString(), randomPort(), - randomCountrySpec()) - test+=gettimestamp() - - for a,p,c in bridgedb.Bridges.parseCountryBlockFile(test.split('\n')): - assert type(a) in (ipaddr.IPv6Address, ipaddr.IPv4Address) - assert isinstance(p, addr.PortList) - assert isinstance(c, list) - assert len(c) > 0 - for y in c: - assert y in countries - #print "address: %s" % a - #print "portlist: %s" % p - #print "countries: %s" % c class BridgeStabilityTests(unittest.TestCase): def setUp(self): @@ -689,7 +651,7 @@ def testSuite(): loader = unittest.TestLoader() for klass in [IPBridgeDistTests, SQLStorageTests, EmailBridgeDistTests, - ParseCountryBlockFileTests, BridgeStabilityTests]: + BridgeStabilityTests]: suite.addTest(loader.loadTestsFromTestCase(klass)) for module in [ bridgedb.Bridges,