March 2013 - tor-commits - lists.torproject.org

[bridgedb/master] 4297 - IPBasedDistributor learns to sort bridges
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 72561ea2b1b30c703efaa8f96c630a13dffd3ed3 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Dec 9 19:01:15 2011 -0800 4297 - IPBasedDistributor learns to sort bridges IPBasedDistributor now uses the FilteredBridgeSplitter to sort bridges into subrings based on the address class of the Bridges' ip(s) (including addresses parsed from the new or-address spec). --- lib/bridgedb/Bridges.py | 17 --------- lib/bridgedb/Dist.py | 91 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 38 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index b0e1fc8..281e5f2 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -791,23 +791,6 @@ class FilteredBridgeSplitter(BridgeHolder): are passed to the constructor as a list of (filterFn, ringName) """ -#XXX: maybe limit the # of cached rings too? -#XXX: keep the bridges around -#XXX: when the # of rings hits RING_LIMIT drop the oldest -#XXX: LRU is probably OK -#XXX: max_cached_rings should be configurable - - -#XXX: need a default ring (f(x): return True) -#XXX: (don't make this a default, we'll just need to have it) - -#XXX: filterRings['NoFilter'] = [list,of,rings] -#XXX: filterRings['Default'] = [list,of,rings] -#XXX: filterRings['IPv6'] = [list,of,rings] - -#XXX: filterRings['more-complex-query-string?'] - - def __init__(self, key, max_cached_rings=3): self.key = key self.filterRings = {} diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index bbff9f1..e2d1559 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -38,35 +38,59 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): def __init__(self, areaMapper, nClusters, key, ipCategories=(), answerParameters=None): self.areaMapper = areaMapper + self.answerParameters = answerParameters self.rings = [] self.categoryRings = [] #DOCDDOC self.categories = [] #DOCDOC + + key2 = bridgedb.Bridges.get_hmac(key, "Assign-Bridges-To-Rings") + key3 = bridgedb.Bridges.get_hmac(key, "Order-Areas-In-Rings") + self.areaOrderHmac = bridgedb.Bridges.get_hmac_fn(key3, hex=False) + key4 = bridgedb.Bridges.get_hmac(key, "Assign-Areas-To-Rings") + self.areaClusterHmac = bridgedb.Bridges.get_hmac_fn(key4, hex=True) + + # add splitter and cache the default rings + # plus leave room for dynamic filters + ring_cache_size = nClusters + len(ipCategories) + 5 + self.splitter = bridgedb.Bridges.FilteredBridgeSplitter(key2, + max_cached_rings=ring_cache_size) + + # assign bridges using a filter function + def filterAssignBridgesToRing(hmac, numRings, assignedRing): + def f(bridge): + digest = hmac(bridge.getID()) + pos = long( digest[:8], 16 ) + which = pos % numRings + if which == assignedRing: return True + return False + return f + for n in xrange(nClusters): key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n) - self.rings.append( bridgedb.Bridges.BridgeRing(key1, - answerParameters) ) - self.rings[-1].setName("IP ring %s"%len(self.rings)) + ring = bridgedb.Bridges.BridgeRing(key1, answerParameters) + ring.setName("IP ring %s"%n) #XXX: should be n+1 for consistency? + + g = filterAssignBridgesToRing(self.splitter.hmac, + nClusters + len(ipCategories), n) + self.splitter.addRing(ring, ring.name, g) + self.rings.append(ring) + n = nClusters for c in ipCategories: logging.info("Building ring: Order-Bridges-In-Ring-%d"%n) key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n) ring = bridgedb.Bridges.BridgeRing(key1, answerParameters) + ring.setName("IP category ring %s"%n) #XXX: should be n+1 for consistency? self.categoryRings.append( ring ) - self.categoryRings[-1].setName( - "IP category ring %s"%len(self.categoryRings)) self.categories.append( (c, ring) ) - n += 1 - key2 = bridgedb.Bridges.get_hmac(key, "Assign-Bridges-To-Rings") - self.splitter = bridgedb.Bridges.FixedBridgeSplitter(key2, - self.rings+self.categoryRings) + g = filterAssignBridgesToRing(self.splitter.hmac, + nClusters + len(ipCategories), n) - key3 = bridgedb.Bridges.get_hmac(key, "Order-Areas-In-Rings") - self.areaOrderHmac = bridgedb.Bridges.get_hmac_fn(key3, hex=False) + self.splitter.addRing(ring, ring.name, g) - key4 = bridgedb.Bridges.get_hmac(key, "Assign-Areas-To-Rings") - self.areaClusterHmac = bridgedb.Bridges.get_hmac_fn(key4, hex=True) + n += 1 def clear(self): self.splitter.clear() @@ -75,7 +99,8 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): """Assign a bridge to this distributor.""" self.splitter.insert(bridge) - def getBridgesForIP(self, ip, epoch, N=1, countryCode=None): + def getBridgesForIP(self, ip, epoch, N=1, countryCode=None, + bridgeFilterRules=None): """Return a list of bridges to give to a user. ip -- the user's IP address, as a dotted quad. epoch -- the time period when we got this request. This can @@ -96,20 +121,44 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): pos = self.areaOrderHmac("category<%s>%s"%(epoch,area)) return ring.getBridges(pos, N, countryCode) - # Which bridge cluster should we look at? - h = int( self.areaClusterHmac(area)[:8], 16) - clusterNum = h % len(self.rings) - ring = self.rings[clusterNum] - # If a ring is empty, consider the next. - while not len(ring): - clusterNum = (clusterNum + 1) % len(self.rings) + # dynamic filter construction + if bridgeFilterRules: + ruleset = frozenset(bridgeFilterRules) + if ruleset in self.splitter.filterRings.keys(): + # cache hit + _,ring = self.splitter.filterRings[ruleset] + else: + # cache miss, add new ring + def filterBridgesByRules(rules): + def g(x): + r = [f(x) for f in rules] + if False in r: return False + return True + return g + # add new ring + #XXX what key do we use here? does it matter? + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, str(bridgeFilterRules)) + ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) + # debug log: cache miss + self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), + populate_from=self.splitter.bridges) + + else: + # Which bridge cluster should we look at? + h = int( self.areaClusterHmac(area)[:8], 16) + clusterNum = h % len(self.rings) ring = self.rings[clusterNum] + # If a ring is empty, consider the next. + while not len(ring): + clusterNum = (clusterNum + 1) % len(self.rings) + ring = self.rings[clusterNum] # Now get the bridge. pos = self.areaOrderHmac("<%s>%s" % (epoch, area)) return ring.getBridges(pos, N) def __len__(self): + #XXX does not include ip categories or filtered rings return sum(len(r) for r in self.rings) def dumpAssignments(self, f, description=""):

1 0

[bridgedb/master] 4297 - Add ipv6 keyword to EmailBasedDistributor
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit c207f38171e122a869c564701950e223f34d4e0a Author: aagbsn <aagbsn(a)extc.org> Date: Thu Dec 15 03:21:59 2011 -0800 4297 - Add ipv6 keyword to EmailBasedDistributor Updates the EmailBasedDistributor to support 'ipv6' keyword in the subject or mail body --- lib/bridgedb/Dist.py | 133 ++++++++++++++++++++++++++++++++----------------- 1 files changed, 87 insertions(+), 46 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index e2d1559..285e088 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -38,7 +38,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): def __init__(self, areaMapper, nClusters, key, ipCategories=(), answerParameters=None): self.areaMapper = areaMapper - self.answerParameters = answerParameters + self.answerParameters = answerParameters self.rings = [] self.categoryRings = [] #DOCDDOC @@ -50,13 +50,15 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): key4 = bridgedb.Bridges.get_hmac(key, "Assign-Areas-To-Rings") self.areaClusterHmac = bridgedb.Bridges.get_hmac_fn(key4, hex=True) - # add splitter and cache the default rings - # plus leave room for dynamic filters + # add splitter and cache the default rings + # plus leave room for dynamic filters ring_cache_size = nClusters + len(ipCategories) + 5 self.splitter = bridgedb.Bridges.FilteredBridgeSplitter(key2, - max_cached_rings=ring_cache_size) + max_cached_rings=ring_cache_size) - # assign bridges using a filter function + logging.debug("added splitter %s" % self.splitter) + + # assign bridges using a filter function def filterAssignBridgesToRing(hmac, numRings, assignedRing): def f(bridge): digest = hmac(bridge.getID()) @@ -69,27 +71,24 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): for n in xrange(nClusters): key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n) ring = bridgedb.Bridges.BridgeRing(key1, answerParameters) - ring.setName("IP ring %s"%n) #XXX: should be n+1 for consistency? + ring.setName("IP ring %s"%n) #XXX: should be n+1 for consistency? - g = filterAssignBridgesToRing(self.splitter.hmac, - nClusters + len(ipCategories), n) - self.splitter.addRing(ring, ring.name, g) - self.rings.append(ring) + g = filterAssignBridgesToRing(self.splitter.hmac, + nClusters + len(ipCategories), n) + self.splitter.addRing(ring, ring.name, g) + self.rings.append(ring) n = nClusters for c in ipCategories: logging.info("Building ring: Order-Bridges-In-Ring-%d"%n) key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n) ring = bridgedb.Bridges.BridgeRing(key1, answerParameters) - ring.setName("IP category ring %s"%n) #XXX: should be n+1 for consistency? + ring.setName("IP category ring %s"%n) #XXX: should be n+1 for consistency? self.categoryRings.append( ring ) self.categories.append( (c, ring) ) - - g = filterAssignBridgesToRing(self.splitter.hmac, - nClusters + len(ipCategories), n) - - self.splitter.addRing(ring, ring.name, g) - + g = filterAssignBridgesToRing(self.splitter.hmac, + nClusters + len(ipCategories), n) + self.splitter.addRing(ring, ring.name, g) n += 1 def clear(self): @@ -100,14 +99,16 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): self.splitter.insert(bridge) def getBridgesForIP(self, ip, epoch, N=1, countryCode=None, - bridgeFilterRules=None): + bridgeFilterRules=None): """Return a list of bridges to give to a user. ip -- the user's IP address, as a dotted quad. epoch -- the time period when we got this request. This can be any string, so long as it changes with every period. N -- the number of bridges to try to give back. """ + logging.debug("getBridgesForIP(%s, %s, %s, %s" % (ip, epoch, N, bridgeFilterRules)) if not len(self.splitter): + logging.debug("bailing without splitter") return [] area = self.areaMapper(ip) @@ -121,29 +122,30 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): pos = self.areaOrderHmac("category<%s>%s"%(epoch,area)) return ring.getBridges(pos, N, countryCode) - # dynamic filter construction - if bridgeFilterRules: - ruleset = frozenset(bridgeFilterRules) - if ruleset in self.splitter.filterRings.keys(): - # cache hit - _,ring = self.splitter.filterRings[ruleset] - else: - # cache miss, add new ring - def filterBridgesByRules(rules): - def g(x): - r = [f(x) for f in rules] - if False in r: return False - return True - return g - # add new ring - #XXX what key do we use here? does it matter? + # dynamic filter construction + if bridgeFilterRules: + logging.debug("bridgeFilterRules: %s" % bridgeFilterRules) + ruleset = frozenset(bridgeFilterRules) + if ruleset in self.splitter.filterRings.keys(): + logging.debug("Cache hit %s" % ruleset) + _,ring = self.splitter.filterRings[ruleset] + else: + logging.debug("Cache miss %s" % ruleset) + def filterBridgesByRules(rules): + def g(x): + r = [f(x) for f in rules] + if False in r: return False + return True + return g + # add new ring + #XXX what key do we use here? does it matter? key1 = bridgedb.Bridges.get_hmac(self.splitter.key, str(bridgeFilterRules)) - ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - # debug log: cache miss - self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), - populate_from=self.splitter.bridges) + ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) + # debug log: cache miss + self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), + populate_from=self.splitter.bridges) - else: + else: # Which bridge cluster should we look at? h = int( self.areaClusterHmac(area)[:8], 16) clusterNum = h % len(self.rings) @@ -158,7 +160,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): return ring.getBridges(pos, N) def __len__(self): - #XXX does not include ip categories or filtered rings + #XXX does not include ip categories or filtered rings return sum(len(r) for r in self.rings) def dumpAssignments(self, f, description=""): @@ -286,14 +288,26 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): self.domainmap = domainmap self.domainrules = domainrules + #XXX cache options not implemented + self.splitter = bridgedb.Bridges.FilteredBridgeSplitter(key2, + max_cached_rings=5) + + #XXX default ring. + def g(x): + return True + + self.splitter.addRing(self.ring, self.ring.name, g) + def clear(self): - self.ring.clear() + self.splitter.clear() + self.ring.clear() # should be take care of by above def insert(self, bridge): """Assign a bridge to this distributor.""" - self.ring.insert(bridge) + self.splitter.insert(bridge) - def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, countryCode=None): + def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, countryCode=None, + bridgeFilterRules=None): """Return a list of bridges to give to a user. emailaddress -- the user's email address, as given in a from line. epoch -- the time period when we got this request. This can @@ -332,7 +346,35 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): db.setWarnedEmail(emailaddress, False) pos = self.emailHmac("<%s>%s" % (epoch, emailaddress)) - result = self.ring.getBridges(pos, N, countryCode) + + ring = None + if bridgeFilterRules: + ruleset = frozenset(bridgeFilterRules) + if ruleset in self.splitter.filterRings.keys(): + logging.debug("Cache hit %s" % ruleset) + _,ring = self.splitter.filterRings[ruleset] + else: + # cache miss, add new ring + logging.debug("Cache miss %s" % ruleset) + def filterBridgesByRules(rules): + def g(x): + r = [f(x) for f in rules] + if False in r: return False + return True + return g + # add new ring + #XXX what key do we use here? does it matter? + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, + str(bridgeFilterRules)) + ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) + # debug log: cache miss + self.splitter.addRing(ring, ruleset, + filterBridgesByRules(bridgeFilterRules), + populate_from=self.splitter.bridges) + else: + ring = self.ring + + result = ring.getBridges(pos, N, countryCode) db.setEmailTime(emailaddress, now) db.commit() @@ -353,5 +395,4 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): db.commit() def dumpAssignments(self, f, description=""): - self.ring.dumpAssignments(f, description) - + self.splitter.dumpAssignments(f, description)

1 0

[bridgedb/master] 4297 - Update BridgeDB's front-end Server.py
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 3e2882752557ff9635da87c3ab439077a0ea3dfa Author: aagbsn <aagbsn(a)extc.org> Date: Thu Dec 15 02:58:35 2011 -0800 4297 - Update BridgeDB's front-end Server.py Front-end's to the HTTPS and email distributors now make use of the address class filtering available in IPBasedDistributor --- lib/bridgedb/Dist.py | 1 + lib/bridgedb/Server.py | 55 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 285e088..afc3592 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -287,6 +287,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): # XXXX clear the store when the period rolls over! self.domainmap = domainmap self.domainrules = domainrules + self.answerParameters = answerParameters #XXX cache options not implemented self.splitter = bridgedb.Bridges.FilteredBridgeSplitter(key2, diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index b67edf2..45cbee0 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -31,6 +31,7 @@ from random import randint from bridgedb.Raptcha import Raptcha import base64 import textwrap +from ipaddr import IPv4Address, IPv6Address try: import GeoIP @@ -147,13 +148,25 @@ class WebResource(twisted.web.resource.Resource): format = request.args.get("format", None) if format and len(format): format = format[0] # choose the first arg + # do want ipv6 support? + ipv6 = False + if "ipv6" in request.postpath: ipv6 = True + if ip: + if ipv6: + rules=[filterBridgesByIP6] + else: + rules=[filterBridgesByIP4] + bridges = self.distributor.getBridgesForIP(ip, interval, self.nBridgesToGive, - countryCode) + countryCode, + bridgeFilterRules=rules) + if bridges: answer = "".join("%s %s\n" % ( - b.getConfigLine(self.includeFingerprints), + b.getConfigLine(self.includeFingerprints,needIPv6=ipv6, + selectFromORAddresses=ipv6), (I18n.BRIDGEDB_TEXT[16] if b.isBlocked(countryCode) else "") ) for b in bridges) else: @@ -379,11 +392,22 @@ def getMailResponse(lines, ctx): # return None,None # Figure out which bridges to send + + # read subject, see if they want ipv6 + ipv6 = False + for ln in lines: + if "ipv6" in ln.strip().lower(): + ipv6 = True + rules=[filterBridgesByIP6] + else: + rules=[filterBridgesByIP4] + try: interval = ctx.schedule.getInterval(time.time()) bridges = ctx.distributor.getBridgesForEmail(clientAddr, interval, ctx.N, - countryCode=None) + countryCode=None, + bridgeFilterRules=rules) except bridgedb.Dist.BadEmail, e: logging.info("Got a mail from a bad email address %r: %s.", clientAddr, e) @@ -433,7 +457,8 @@ def getMailResponse(lines, ctx): if bridges: with_fp = ctx.cfg.EMAIL_INCLUDE_FINGERPRINTS - answer = "".join(" %s\n" % b.getConfigLine(with_fp) for b in bridges) + answer = "".join(" %s\n" % b.getConfigLine(with_fp, needIPv6=ipv6,\ + selectFromORAddresses=ipv6) for b in bridges) else: answer = "(no bridges currently available)" @@ -646,3 +671,25 @@ def getCCFromRequest(request): if len(path) == 2: return path.lower() return None + +def filterBridgesByIP4(bridge): + try: + if IPv4Address(bridge.ip): return True + except ValueError: + pass + + for k in bridge.or_addresses.keys(): + if type(k) is IPv4Address: + return True + return False + +def filterBridgesByIP6(bridge): + try: + if IPv6Address(bridge.ip): return True + except ValueError: + pass + + for k in bridge.or_addresses.keys(): + if type(k) is IPv6Address: + return True + return False

1 0

[bridgedb/master] 4297 - add Tests for BridgeDB IPv6 upgrades
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 2e5f4b8c84350342229357dbfa051c71f376b6e9 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Dec 6 19:08:15 2011 -0800 4297 - add Tests for BridgeDB IPv6 upgrades Test descriptor file parsing, or-address, and mixed address class (IPv4 and IPv6) or-addresses. Tests for IPBasedDistributor and FilteredBridgeSplitter --- lib/bridgedb/Tests.py | 245 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 files changed, 230 insertions(+), 15 deletions(-) diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 9648f86..ead36fc 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -17,6 +17,7 @@ import bridgedb.Dist import bridgedb.Time import bridgedb.Storage import re +import ipaddr def suppressWarnings(): warnings.filterwarnings('ignore', '.*tmpnam.*') @@ -24,22 +25,73 @@ def suppressWarnings(): def randomIP(): return ".".join([str(random.randrange(1,256)) for _ in xrange(4)]) +def randomIP6(): + return "%s" % ipaddr.IPAddress(random.getrandbits(128)) + def random16IP(): upper = "123.123." # same 16 lower = ".".join([str(random.randrange(1,256)) for _ in xrange(2)]) return upper+lower -def fakeBridge(orport=8080): +def randomPort(): + return str(random.randint(1,65535)) + +def randomPortSpec(): + """ + returns a random list of ports and port ranges (strings) + ranges are portlow-porthigh + """ + ports = [] + for i in range(0,24): + ports.append(random.randint(1,65535)) + ports.sort(reverse=True) + + ports = [str(x) for x in ports] + + ranges = [] + for i in range(0,8): + ranges.append("%s-%s" % (ports.pop(),ports.pop())) + ports.extend(ranges) + + portspec = "" + for i in range(0,16): + portspec += "%s," % random.choice(ports) + portspec = portspec.rstrip(',') #remove trailing , + return portspec + +def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False): nn = "bridge-%s"%random.randrange(0,1000000) ip = randomIP() fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) - return bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + b.setStatus(running, stable) + + if or_addresses: + for i in xrange(0,8): + b.or_addresses[ipaddr.IPv4Address(randomIP())] =\ + bridgedb.Bridges.PortList(randomPortSpec()) + return b + +def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False): + nn = "bridge-%s"%random.randrange(0,1000000) + ip = randomIP6() + fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) + b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + b.setStatus(running, stable) -def fake16Bridge(orport=8080): + if or_addresses: + for i in xrange(0,8): + b.or_addresses[ipaddr.IPv6Address(randomIP6())] =\ + bridgedb.Bridges.PortList(randomPortSpec()) + return b + +def fake16Bridge(orport=8080, running=True, stable=True): nn = "bridge-%s"%random.randrange(0,1000000) ip = random16IP() fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) - return bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + b.setStatus(running, stable) + return b class RhymesWith255Category: def contains(self, ip): @@ -130,18 +182,103 @@ class IPBridgeDistTests(unittest.TestCase): self.assertEquals(len(fps), 5) self.assertTrue(count >= 1) - def testDistWithFilter16(self): + #def testDistWithFilter16(self): + # d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + # for _ in xrange(256): + # d.insert(fake16Bridge()) + # n = d.getBridgesForIP("1.2.3.4", "x", 10) + + # slash16s = dict() + # for bridge in n: + # m = re.match(r'(\d+\.\d+)\.\d+\.\d+', bridge.ip) + # upper16 = m.group(1) + # self.assertTrue(upper16 not in slash16s) + # slash16s[upper16] = True + + def testDistWithFilterIP6(self): d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") - for _ in xrange(256): - d.insert(fake16Bridge()) - n = d.getBridgesForIP("1.2.3.4", "x", 10) + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) - slash16s = dict() - for bridge in n: - m = re.match(r'(\d+\.\d+)\.\d+\.\d+', bridge.ip) - upper16 = m.group(1) - self.assertTrue(upper16 not in slash16s) - slash16s[upper16] = True + def filterBridgesByIP6(bridge): + for k in bridge.or_addresses.keys(): + if type(k) is ipaddr.IPv6Address: + return True + return False + + for i in xrange(500): + b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[filterBridgesByIP6]) + assert filterBridgesByIP6(random.choice(b)) + + def testDistWithFilterIP4(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) + + def filterBridgesByIP4(bridge): + for k in bridge.or_addresses.keys(): + if type(k) is ipaddr.IPv4Address: + return True + return False + + for i in xrange(500): + b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[filterBridgesByIP4]) + assert filterBridgesByIP4(random.choice(b)) + + def testDistWithFilterBoth(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) + + def filterBridgesByIP4(bridge): + for k in bridge.or_addresses.keys(): + if type(k) is ipaddr.IPv4Address: + return True + return False + + def filterBridgesByIP6(bridge): + for k in bridge.or_addresses.keys(): + if type(k) is ipaddr.IPv6Address: + return True + return False + + for i in xrange(50): + b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[ + filterBridgesByIP4, filterBridgesByIP6]) + if b: + t = b.pop() + assert filterBridgesByIP4(t) + assert filterBridgesByIP6(t) + + def testDistWithFilterAll(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(250): + d.insert(fakeBridge6(or_addresses=True)) + d.insert(fakeBridge(or_addresses=True)) + + def filterBridgesByOnlyIP4(bridge): + for k in bridge.or_addresses.keys(): + if type(k) is ipaddr.IPv6Address: + return False + if type(k) is ipaddr.IPv4Address: + return True + return False + + def filterBridgesByOnlyIP6(bridge): + for k in bridge.or_addresses.keys(): + if type(k) is ipaddr.IPv4Address: + return False + if type(k) is ipaddr.IPv6Address: + return True + return False + + for i in xrange(5): + b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[ + filterBridgesByOnlyIP4, filterBridgesByOnlyIP6]) + assert len(b) == 0 class DictStorageTests(unittest.TestCase): def setUp(self): @@ -296,11 +433,89 @@ class SQLStorageTests(unittest.TestCase): db.cleanWarnedEmails(t+200) self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) +class ParseDescFileTests(unittest.TestCase): + def testSimpleDesc(self): + simpleDesc = "router Unnamed %s %s 0 9030\n"\ + "opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ + "opt @purpose bridge\n"\ + "router-signature\n" + test = "" + + for i in range(100): + test+= "".join(simpleDesc % (randomIP(), randomPort())) + + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + self.assertEquals(len(bs), 100) + + for b in bs: + b.assertOK() + + def testSingleOrAddress(self): + simpleDesc = "router Unnamed %s %s 0 9030\n"\ + "opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ + "opt @purpose bridge\n" + orAddress = "or-address %s:%s\n" + test = "" + + for i in range(100): + test+= simpleDesc % (randomIP(), randomPort()) + test+= orAddress % (randomIP(),randomPort()) + test+= "router-signature\n" + + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + self.assertEquals(len(bs), 100) + + for b in bs: + b.assertOK() + + def testMultipleOrAddress(self): + simpleDesc = "router Unnamed %s %s 0 9030\n"\ + "opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ + "opt @purpose bridge\n" + orAddress = "or-address %s:%s\n" + test = "" + + for i in range(100): + test+= simpleDesc % (randomIP(), randomPort()) + for i in xrange(4): + test+= orAddress % (randomIP(),randomPortSpec()) + for i in xrange(4): + test+= orAddress % ("[%s]" % randomIP6(),randomPortSpec()) + test+= "router-signature\n" + + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + self.assertEquals(len(bs), 100) + + for b in bs: + b.assertOK() + + def testConvolutedOrAddress(self): + simpleDesc = "router Unnamed %s %s 0 9030\n"\ + "opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ + "opt @purpose bridge\n" + orAddress = "or-address %s:%s\n" + test = "" + + for i in range(100): + test+= simpleDesc % (randomIP(), randomPort()) + for i in xrange(4): + test+= orAddress % (randomIP(),randomPortSpec()) + for i in xrange(4): + test+= orAddress % ("[%s]" % randomIP6(),randomPortSpec()) + test+= "router-signature\n" + + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + self.assertEquals(len(bs), 100) + + for b in bs: + b.assertOK() + def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader() - for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests, EmailBridgeDistTests ]: + for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests, + EmailBridgeDistTests, ParseDescFileTests ]: suite.addTest(loader.loadTestsFromTestCase(klass)) for module in [ bridgedb.Bridges,

1 0

[bridgedb/master] 4297 - Warn about bridge pools + dynamic filtering
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit ee4d719b3dcd99ab60c6d1db14302abe62667959 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jan 9 11:04:17 2012 -0800 4297 - Warn about bridge pools + dynamic filtering If using dynamic bridge filtering, the IPBasedDistributor must be suppled wth a filtering function that will select the correct pool for the requesting IP. The filter could be implemented in a similar fashion as the existing approach, which is to map the requesting ip address to an area and assign then map the areas to pools of bridges. An alternate approach would define bridge pools by a set of corresponding country codes. The pools may either be of roughly equal size to simplify bridge assignment, or weighted by the aggregate bridge users from those countries. Presently, BridgeDB does neither. There are very few IPv6 bridges, so bypassing bridge clustering should be OK for the short term. Deliverables for Sponsor E require that BridgeDB be able to reassign blocked bridges to other countries; the extensions to BridgeDB required to implement this policy will replace the current clusters with bridge pools that correspond to geographic regions and address the current filter limitations. --- lib/bridgedb/Dist.py | 4 ++++ lib/bridgedb/Server.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index afc3592..6a459be 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -123,8 +123,12 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): return ring.getBridges(pos, N, countryCode) # dynamic filter construction + #XXX: may bypass clusters! front-end must supply clustering function + # or provide another mechanism (perhaps geoip based) to sort and select + # bridges from separate pools. if bridgeFilterRules: logging.debug("bridgeFilterRules: %s" % bridgeFilterRules) + #XXX: is there a better way to cache by ruleset signature? ruleset = frozenset(bridgeFilterRules) if ruleset in self.splitter.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 45cbee0..6db4522 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -154,9 +154,13 @@ class WebResource(twisted.web.resource.Resource): if ip: if ipv6: + #XXX: this ruleset bypasses areamapper and bridge clusters. + # for now, there are very few IPv6 bridges. rules=[filterBridgesByIP6] else: - rules=[filterBridgesByIP4] + #XXX: default to areamapper + cluster, does not use oraddress + rules = None + #rules=[filterBridgesByIP4] bridges = self.distributor.getBridgesForIP(ip, interval, self.nBridgesToGive,

1 0

[bridgedb/master] 4297 - ipv6 uniformMap
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 3bee35c8d3977d0645bd57b8fc7bf4ef003538af Author: aagbsn <aagbsn(a)extc.org> Date: Tue Feb 28 13:21:50 2012 -0800 4297 - ipv6 uniformMap Adds an ipv6-comptabile uniform mapper. --- lib/bridgedb/Dist.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 6a459be..95fe0b5 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -12,6 +12,7 @@ import bridgedb.Storage import logging import re import time +from ipaddr import IPv6Address, IPAddress def uniformMap(ip): """Map an IP to an arbitrary 'area' string, such that any two /24 addresses @@ -20,7 +21,10 @@ def uniformMap(ip): >>> uniformMap('1.2.3.4') '1.2.3' """ - return ".".join( ip.split(".")[:3] ) + if type(IPAddress(ip)) is IPv6Address: + return ":".join(IPv6Address(ip).exploded.split(':')[:4]) + else: + return ".".join( ip.split(".")[:3] ) class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): """Object that hands out bridges based on the IP address of an incoming

1 0

[bridgedb/master] 4297 - Exception Oddities
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit c04f4035fbadc48872ac9f0cc30fe69019af3e8e Author: aagbsn <aagbsn(a)extc.org> Date: Tue Feb 28 14:22:05 2012 -0800 4297 - Exception Oddities Some versions (tested with 2.6.6) of Python exhibit odd exception handling behavior; this fix corrects an issue where the parent class exception handler was catching exceptions meant for a subclassed exception handler --- lib/bridgedb/Server.py | 17 +++++++++-------- 1 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 6db4522..a2460b8 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -32,6 +32,7 @@ from bridgedb.Raptcha import Raptcha import base64 import textwrap from ipaddr import IPv4Address, IPv6Address +from bridgedb.Dist import BadEmail, TooSoonEmail, IgnoreEmail try: import GeoIP @@ -364,7 +365,7 @@ def getMailResponse(lines, ctx): try: _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower()) - except bridgedb.Dist.BadEmail: + except BadEmail: logging.info("Ignoring bad address on incoming email.") return None,None if not addrdomain: @@ -412,13 +413,8 @@ def getMailResponse(lines, ctx): interval, ctx.N, countryCode=None, bridgeFilterRules=rules) - except bridgedb.Dist.BadEmail, e: - logging.info("Got a mail from a bad email address %r: %s.", - clientAddr, e) - return None, None - # Handle rate limited email - except bridgedb.Dist.TooSoonEmail, e: + except TooSoonEmail, e: logging.info("Got a mail too frequently; warning %r: %s.", clientAddr, e) @@ -441,11 +437,16 @@ def getMailResponse(lines, ctx): f.seek(0) return clientAddr, f - except bridgedb.Dist.IgnoreEmail, e: + except IgnoreEmail, e: logging.info("Got a mail too frequently; ignoring %r: %s.", clientAddr, e) return None, None + except BadEmail, e: + logging.info("Got a mail from a bad email address %r: %s.", + clientAddr, e) + return None, None + # Generate the message. f = StringIO() w = MimeWriter.MimeWriter(f)

1 0

[bridgedb/master] 5367 - Fix python gotcha in __init__
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit aba290f8e3a1eaa38bfd580d350e0721911b49da Author: aagbsn <aagbsn(a)extc.org> Date: Sun Mar 11 22:01:28 2012 -0700 5367 - Fix python gotcha in __init__ Python default arguments are only parsed once, therefore the list was shared between all instances. See: http://www.ferg.org/projects/python_gotchas.html --- lib/bridgedb/Server.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index a1b837a..b67edf2 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -50,7 +50,7 @@ class WebResource(twisted.web.resource.Resource): def __init__(self, distributor, schedule, N=1, useForwardedHeader=False, includeFingerprints=True, useRecaptcha=False,recaptchaPrivKey='', recaptchaPubKey='', - domains=[]): + domains=None): """Create a new WebResource. distributor -- an IPBasedDistributor object schedule -- an IntervalSchedule object @@ -63,6 +63,9 @@ class WebResource(twisted.web.resource.Resource): self.nBridgesToGive = N self.useForwardedHeader = useForwardedHeader self.includeFingerprints = includeFingerprints + + # do not use mutable types as __init__ defaults! + if not domains: domains = [] self.domains = domains # recaptcha options

1 0

[bridgedb/master] 4297 - add draft proposal to repo
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit ca16efcefe2b66d4db4de0fab990d2bba27f3e1a Author: aagbsn <aagbsn(a)extc.org> Date: Tue Dec 6 13:25:20 2011 -0800 4297 - add draft proposal to repo This is the draft proposal that was sent to tor-dev on Dec 5, 2011 plus the addition of the new SQL schema for storing or-addresses. --- xxx-bridgedb-learns-ipv6.txt | 280 ++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 280 insertions(+), 0 deletions(-) diff --git a/xxx-bridgedb-learns-ipv6.txt b/xxx-bridgedb-learns-ipv6.txt new file mode 100644 index 0000000..44191b6 --- /dev/null +++ b/xxx-bridgedb-learns-ipv6.txt @@ -0,0 +1,280 @@ +Filename: xxx-bridgedb-learns-ipv6.txt +Title: BridgeDB Learns IPv6 +Author: Aaron Gibson +Created: 5 Dec 2011 +Status: Draft + +Overview: + + This document outlines what we'll do to make BridgeDB fully support IPv6 + bridges, and fully support IPv6 with the email, https, and bucket + distributors. + +Motivation: + + IPv6 bridges need a BridgeDB too. + +What needs to change: + + There are two main tasks that must be completed for BridgeDB to support IPv6. + + 1. BridgeDB must be able to parse IPv6 addresses from router descriptors. + (Currently, BridgeDB does not recognize the or-address line described in + 186-multiple-orports.txt) + + 2. BridgeDB must decide how to hand out IPv6 addresses. (Currently, + BridgeDB distributors are not IPv6 aware, and provide no support for + distinguishing bridges by address class) + + +1. BridgeDB learns to parse or-address + + BridgeDB must learn how to parse the new or-address line from server + descriptors. The new or-address line allows a router to specify a list of + addresses and ports or port-ranges. + + Here is the or-address specification (see: 186-multiple-orports.txt) + + or-address SP ADDRESS ":" PORTLIST NL + ADDRESS = IP6ADDR | IP4ADDR + IPV6ADDR = an ipv6 address, surrounded by square brackets. + IPV4ADDR = an ipv4 address, represented as a dotted quad. + PORTLIST = PORTSPEC | PORTSPEC "," PORTLIST + PORTSPEC = PORT | PORT "-" PORT + PORT = a number between 1 and 65535 inclusive. + + BridgeDB must now comprehend and store multiple listening addresses and + ports. BridgeDB currently assumes that each bridge has only one listen + address. BridgeDB must be modified to take one of the following approaches: + + a. Treat each ADDRESS:PORT combination as a separate bridge entity + b. Display a subset of each bridges ADDRESS:PORT entries in a response + c. Display all of each bridges ADDRESS:PORT entries in a response + + Given any address of the bridge you can learn its fingerprint, and use that + to look up its descriptor at tonga and learn the rest of the addresses. so + counting a bridge with 5 addresses as 5 bridges makes it more likely to get + blocked by a smart adversary. but more useful against a weaker adversary. + #XXX: thanks arma! + # any other thoughts here? option c. seems to be the simplest to implement. + + BridgeDB should be able to mark specific IP:port pairs as blocked, and + indicate where it is blocked (e.g. by country code). This requirement is + complicated by the fact that or-address may specify a range of listening + ports. + + How are IPv6 Addresses stored in BridgeDB? + + IPv6 Addresses are stored as strings, the same way as IPv4 addresses. + #XXX: is this better than using the ipaddr.IPAddress class? + + As the new or-address specification allows for multiple ADDRESS:PORT + BridgeDB's persistent database must also be modified. + + A new table 'or-address' shall be created with the following fields: + ex. from updated BridgeDB schema: + + CREATE TABLE BridgeOrAddresses ( + id INTEGER PRIMARY KEY NOT NULL, + hex_key, + address, + or_port, + address_class, + ); + + CREATE INDEX BridgeOrAddressesKeyIndex on BridgeOrAddresses ( hex_key ); + + How are Bridges differentiated by address class? + + Bridges are differentiated by the string representation of their IP + address. + + When BridgeDB needs to make a distinction between IP address classes, the + python module ipaddr-py (https://code.google.com/p/ipaddr-py/) will be + used to determine address class. + +2. BridgeDB learns how to selectively distribute IPv6 bridges + + BridgeDB's 3 distributors must be able to selectively provide both + IPv4 and/or IPv6 bridges to clients. Deployment of these distributors must + take care to mitigate reachability issues due partly to the ongoing + transition from IPv4 to IPv6. + + [One such issue is clients who have IPv6 support on their local network but + the client's ISP does not; such a client may try to reach the IPv6 address + specified by a AAAA record and fail to connect.] + + The 3 distributor types that BridgeDB currently features are: + + 1. HTTPS Distributor + + The HTTPS distributor must be able to selectively offer both IPv4 and + IPv6 bridges to its' clients, and BridgeDB must support both IPv4 and + IPv6 connections. + + #XXX the twisted framework does not currently support ipv6. However, + # it is possible to place BridgeDB behind a forwarding proxy such as + # apache or nginx, which will pass the client address to BridgeDB in the + # X_FORWARDED_FOR header. BridgeDB HTTPS distributor must be able to + # parse the X_FORWARDED_FOR header for both IPv4 and IPv6 addresses. + # Additionally, BridgeDB should eventually support IPv6 natively when + # the twisted framework provides adequate IPv6 support. + + How does bridgedb determine whether to respond with ipv4 or ipv6 + bridges? + + Users select IPv4 or IPv6 bridges by visiting different URLs. An + informational message added to the BridgeDB response will include the + other URL. Two separate BridgeDB instances are run, one for each URL. + + Alternately, ipv6 bridges could be requested by visiting + bridges.tpo/ipv6 or similar URL path scheme. + + Proposed Additional Hostname For IPv6 Bridges + + BridgeDB shall listen for requests on two different hostnames, + bridges.torproject.org and bridgesv6.torproject.org. + + DNS Configuration Details + + bridges.torproject.org shall not have an AAAA record until the + addition of the record is determined to be sound. + + bridgesv6.torproject.org shall have both an AAAA and A record. + + This is to avoid the confused-client scenario described above. + + How does BridgeDB know which URL was requested? + + This section describes how BridgeDB could be modified to support + requests to both bridges.torproject.org and bridgesv6.torproject.org + with a single BridgeDB instance. + + A single BridgeDB instance could handle requests to both + bridges.torproject.org and bridgesv6.torproject.org by checking the + Host header sent by the browser. The Host header is optional. In + order to expose the selector explitely BridgeDB must check the query + string for the following parameters: + + q=ipv4 - Request IPv4 bridges. + q=ipv6 - Request IPv6 bridges. + + Parameters may be repeated to select multiple classes, e.g. + + q=ipv4&q=ipv6 - Request both IPv4 and IPv6 bridges. + + When no parameters are set, by default BridgeDB must return addresses + of the same class as the client. This default may promote IPv6 use + where possible. + + How does someone end up at bridgesv6.torproject.org? + + BridgeDB should include a message at the end of its' response. + e.g. + + "Get IPv4 bridges https://bridges.torproject.org" + "Get IPv6 bridges from https://bridgesv6.torproject.org" + "You must have IPv6 for these bridges to work." + #XXX: will users understand what this means? + + How does IPv6 affect address datamining of https distribution? + A user may be allocated a /128, or a /64. + An adversary may control a /32 or perhaps larger + Proposal: Enable reCAPTCHA support by default. + + How do IPv6 addresses work with the IPBasedDistributor? + #XXX: I need feedback on this + # do we use all 128 bits here? + # upper N bits? lower N bits? random or specific N bits? + + How are IPv6 Bridges actually distinguished? + + A new type of BridgeSplitter (sort of like a BridgeHolder) + is devised; the function of which is to split bridges into different + rings determined by a filter function. + + The filtering mechanism here is similar to BridgeDB's ipCategories + implementation, the difference is that both the filters and ring + names are specified at instance construction. + + The construction of a BridgeSplitter instance should be done by + passing lists of tuples (ringName,filterFunction) to the constructor. + This feature allows for dynamically creating filtered BridgeRings, + which would prove useful for constructing more complex filters, for + example, filtering by both address class and reachability from + specific countries. + + This implementation could increase the rate at which bridges are + detected and blocked, although the rate could be controlled by the + frequency that BridgeDB updates its knowledge of blocked bridges. + + #XXX: I have some concern about the performance of + # filtering bridges dynamically for each response. BridgeDB should + # learn to cache recently used dynamic filters so that the impact of + # popular requests will be reduced, and BridgeDB does not have to + # pre-compute or identify which types of requests will be popular. + + The implementation could look similar to the current 'subring' + implementation; or the current 'ipCategories' implementation. Both of + the features are implemented using subrings that hold a subset of + the parent ring's bridges; the subset being defined by a filtering + function. + + An accompanying Distributor based on the existing IPBasedDistributor + shall be designed to use the above BridgeSplitter so that sorted + Bridges are selectable by address type. Because a bridge + may now have both IPv6 and IPv4 addresses, BridgeDB needs to take + one of the following approaches when only a single address class is + requested: + + a. filter addresses of the other address class from the response + b. include the other addresses in the response + + 2. Email Distributor + + The Email Distributor must accept additional new commands parsed from + the subject or a single line in the body of an email message. + + ipv4 - request IPv4 bridges. + ipv6 - request IPv6 bridges. + + The default action may be set in bridgedb.conf with the option + EMAIL_DEFAULT_ADDRESS_CLASS, which must be one of 'ipv6' or 'ipv4'. If + the option is not given in the config, EMAIL_DEFAULT_ADDRESS_CLASS shall + default to 'ipv4'. + + Similar to the IPBasedDistributor, BridgeDB must determine how the + response should accommodate bridges with both address classes. + + 3. Unassigned Distributor and Buckets + + BridgeDB must provide a selector to choose between exporting + IPv4, IPv6, or both types of bridges. + + BridgeDB currently provides a way to export bucket files with + unallocated bridges. The existing syntax provides no mechanism to + differentiate by address class. + + Proposed new FILE_BUCKET syntax: + + A dictionary of dictionaries with the following acceptable keys and + values. + + 'filename_prefix' shall be a unique string used as the output filename + prefix. This is string is also the key to a dictionary that contains + the following key/values: + + 'address-class' : one of either 'ipv6' or 'ipv4' + 'number' : an integer > 0 + + Users may wish to provide descriptive names, + e.g. + + FILE_BUCKETS = { + 'filename_prefix': {'address-class': 'ipv6', 'number': 10}, + 'descriptive_name': {'address-class': 'ipv6', 'number': 10}, + } + + Future BridgeDB enhancements may expand these options to include other + filters. + #XXX: e.g. buckets of bridges 'not-blocked-in'

1 0

[bridgedb/master] 4297 - Move filter functions to Filters.py
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 1bcaf8328e5285b3648f9714d0cd15d3ba183d32 Author: aagbsn <aagbsn(a)extc.org> Date: Thu May 24 16:20:54 2012 -0700 4297 - Move filter functions to Filters.py --- lib/bridgedb/Dist.py | 30 ++++------------------- lib/bridgedb/Filters.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++ lib/bridgedb/Server.py | 25 ++----------------- lib/bridgedb/Tests.py | 45 ++++------------------------------- 4 files changed, 73 insertions(+), 86 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index d393ee2..c151bbc 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -14,6 +14,9 @@ import re import time from ipaddr import IPv6Address, IPAddress +from bridgedb.Filters import filterAssignBridgesToRing +from bridgedb.Filters import filterBridgesByRules + def uniformMap(ip): """Map an IP to an arbitrary 'area' string, such that any two /24 addresses get the same string. @@ -54,24 +57,15 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): key4 = bridgedb.Bridges.get_hmac(key, "Assign-Areas-To-Rings") self.areaClusterHmac = bridgedb.Bridges.get_hmac_fn(key4, hex=True) - # add splitter and cache the default rings - # plus leave room for dynamic filters + # add splitter and cache the default rings + # plus leave room for dynamic filters ring_cache_size = nClusters + len(ipCategories) + 5 self.splitter = bridgedb.Bridges.FilteredBridgeSplitter(key2, max_cached_rings=ring_cache_size) logging.debug("added splitter %s" % self.splitter) - # assign bridges using a filter function - def filterAssignBridgesToRing(hmac, numRings, assignedRing): - def f(bridge): - digest = hmac(bridge.getID()) - pos = long( digest[:8], 16 ) - which = pos % numRings - if which == assignedRing: return True - return False - return f - + # assign bridges using a filter function for n in xrange(nClusters): key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n) ring = bridgedb.Bridges.BridgeRing(key1, answerParameters) @@ -139,12 +133,6 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): _,ring = self.splitter.filterRings[ruleset] else: logging.debug("Cache miss %s" % ruleset) - def filterBridgesByRules(rules): - def g(x): - r = [f(x) for f in rules] - if False in r: return False - return True - return g # add new ring #XXX what key do we use here? does it matter? key1 = bridgedb.Bridges.get_hmac(self.splitter.key, str(bridgeFilterRules)) @@ -365,12 +353,6 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): else: # cache miss, add new ring logging.debug("Cache miss %s" % ruleset) - def filterBridgesByRules(rules): - def g(x): - r = [f(x) for f in rules] - if False in r: return False - return True - return g # add new ring #XXX what key do we use here? does it matter? key1 = bridgedb.Bridges.get_hmac(self.splitter.key, diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py new file mode 100644 index 0000000..72e9004 --- /dev/null +++ b/lib/bridgedb/Filters.py @@ -0,0 +1,59 @@ +# BridgeDB by Nick Mathewson. +# Copyright (c) 2007-2012, The Tor Project, Inc. +# See LICENSE for licensing information + +from ipaddr import IPv6Address, IPv4Address + +def filterAssignBridgesToRing(hmac, numRings, assignedRing): + def f(bridge): + digest = hmac(bridge.getID()) + pos = long( digest[:8], 16 ) + which = pos % numRings + if which == assignedRing: return True + return False + return f + +def filterBridgesByRules(rules): + def g(x): + r = [f(x) for f in rules] + if False in r: return False + return True + return g + +def filterBridgesByIP4(bridge): + try: + if IPv4Address(bridge.ip): return True + except ValueError: + pass + + for k in bridge.or_addresses.keys(): + if type(k) is IPv4Address: + return True + return False + +def filterBridgesByIP6(bridge): + try: + if IPv6Address(bridge.ip): return True + except ValueError: + pass + + for k in bridge.or_addresses.keys(): + if type(k) is IPv6Address: + return True + return False + +def filterBridgesByOnlyIP4(bridge): + for k in bridge.or_addresses.keys(): + if type(k) is IPv6Address: + return False + if type(k) is IPv4Address: + return True + return False + +def filterBridgesByOnlyIP6(bridge): + for k in bridge.or_addresses.keys(): + if type(k) is IPv4Address: + return False + if type(k) is IPv6Address: + return True + return False diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index a2460b8..2de1289 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -33,6 +33,9 @@ import base64 import textwrap from ipaddr import IPv4Address, IPv6Address from bridgedb.Dist import BadEmail, TooSoonEmail, IgnoreEmail + +from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.Filters import filterBridgesByIP4 try: import GeoIP @@ -676,25 +679,3 @@ def getCCFromRequest(request): if len(path) == 2: return path.lower() return None - -def filterBridgesByIP4(bridge): - try: - if IPv4Address(bridge.ip): return True - except ValueError: - pass - - for k in bridge.or_addresses.keys(): - if type(k) is IPv4Address: - return True - return False - -def filterBridgesByIP6(bridge): - try: - if IPv6Address(bridge.ip): return True - except ValueError: - pass - - for k in bridge.or_addresses.keys(): - if type(k) is IPv6Address: - return True - return False diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index ead36fc..d989f7d 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -19,6 +19,11 @@ import bridgedb.Storage import re import ipaddr +from bridgedb.Filters import filterBridgesByIP4 +from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.Filters import filterBridgesByOnlyIP4 +from bridgedb.Filters import filterBridgesByOnlyIP6 + def suppressWarnings(): warnings.filterwarnings('ignore', '.*tmpnam.*') @@ -201,12 +206,6 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) - def filterBridgesByIP6(bridge): - for k in bridge.or_addresses.keys(): - if type(k) is ipaddr.IPv6Address: - return True - return False - for i in xrange(500): b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[filterBridgesByIP6]) assert filterBridgesByIP6(random.choice(b)) @@ -217,12 +216,6 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) - def filterBridgesByIP4(bridge): - for k in bridge.or_addresses.keys(): - if type(k) is ipaddr.IPv4Address: - return True - return False - for i in xrange(500): b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[filterBridgesByIP4]) assert filterBridgesByIP4(random.choice(b)) @@ -233,18 +226,6 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) - def filterBridgesByIP4(bridge): - for k in bridge.or_addresses.keys(): - if type(k) is ipaddr.IPv4Address: - return True - return False - - def filterBridgesByIP6(bridge): - for k in bridge.or_addresses.keys(): - if type(k) is ipaddr.IPv6Address: - return True - return False - for i in xrange(50): b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[ filterBridgesByIP4, filterBridgesByIP6]) @@ -259,22 +240,6 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) - def filterBridgesByOnlyIP4(bridge): - for k in bridge.or_addresses.keys(): - if type(k) is ipaddr.IPv6Address: - return False - if type(k) is ipaddr.IPv4Address: - return True - return False - - def filterBridgesByOnlyIP6(bridge): - for k in bridge.or_addresses.keys(): - if type(k) is ipaddr.IPv4Address: - return False - if type(k) is ipaddr.IPv6Address: - return True - return False - for i in xrange(5): b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[ filterBridgesByOnlyIP4, filterBridgesByOnlyIP6])

1 0