March 2013 - tor-commits - lists.torproject.org

[bridgedb/master] 4297 - Update tests for IP categories
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 177ea49d2e9d168f98f38832719efdda4d38d995 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Mar 12 03:58:31 2012 -0700 4297 - Update tests for IP categories Rings are set up a bit differently now, and broke this test. --- lib/bridgedb/Tests.py | 27 ++++++++++++++++----------- 1 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index d989f7d..067eafa 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -151,20 +151,25 @@ class IPBridgeDistTests(unittest.TestCase): def testDistWithCategories(self): d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo", [RhymesWith255Category()]) - assert len(d.categoryRings) == 1 - rhymesWith255Ring = d.categoryRings[0] + assert len(d.categories) == 1 for _ in xrange(256): d.insert(fakeBridge()) - # Make sure this IP doesn't get any rhymes-with-255 bridges - n = d.getBridgesForIP("1.2.3.4", "x", 10) - for b in n: - self.assertFalse(b.getID() in rhymesWith255Ring.bridgesByID) - - # Make sure these IPs all get rhymes-with-255 bridges - for ip in ("6.7.8.255", "10.10.10.255"): - n = d.getBridgesForIP("1.2.3.255", "xyz", 10) + + for _ in xrange(256): + # Make sure that the categories do not overlap + f = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(4)]) + g = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(3)] + ['255']) + n = d.getBridgesForIP(g(), "x", 10) + n2 = d.getBridgesForIP(f(), "x", 10) + + assert(len(n) > 0) + assert(len(n2) > 0) + for b in n: - self.assertTrue(b.getID() in rhymesWith255Ring.bridgesByID) + assert (b not in n2) + + for b in n2: + assert (b not in n) def testDistWithPortRestrictions(self): param = bridgedb.Bridges.BridgeRingParameters(needPorts=[(443, 1)])

1 0

[bridgedb/master] 4297 - Cache dynamic filter functions
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit a91cf041a0c5b3ad9a81a26162794d382a2a10f9 Author: aagbsn <aagbsn(a)extc.org> Date: Thu May 24 16:34:26 2012 -0700 4297 - Cache dynamic filter functions --- lib/bridgedb/Filters.py | 38 ++++++++++++++++++++++++++------------ 1 files changed, 26 insertions(+), 12 deletions(-) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index 72e9004..fd79704 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -4,21 +4,35 @@ from ipaddr import IPv6Address, IPv4Address +funcs = {} + def filterAssignBridgesToRing(hmac, numRings, assignedRing): - def f(bridge): - digest = hmac(bridge.getID()) - pos = long( digest[:8], 16 ) - which = pos % numRings - if which == assignedRing: return True - return False - return f + ruleset = frozenset([hmac, numRings, assignedRing]) + try: + return funcs[ruleset] + except KeyError: + def f(bridge): + digest = hmac(bridge.getID()) + pos = long( digest[:8], 16 ) + which = pos % numRings + if which == assignedRing: return True + return False + f.__name__ = "filterAssignBridgesToRing(%s, %s, %s)" % (hmac, numRings, + assignedRing) + funcs[ruleset] = f + return f def filterBridgesByRules(rules): - def g(x): - r = [f(x) for f in rules] - if False in r: return False - return True - return g + ruleset = frozenset(rules) + try: + return funcs[ruleset] + except KeyError: + def g(x): + r = [f(x) for f in rules] + if False in r: return False + return True + funcs[ruleset] = g + return g def filterBridgesByIP4(bridge): try:

1 0

[bridgedb/master] 4297 - add IPBasedDistributor IP Clustering Filter
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit f6c7ee872671995eb32dfb790542932f5279206c Author: aagbsn <aagbsn(a)extc.org> Date: Fri May 25 16:36:25 2012 -0700 4297 - add IPBasedDistributor IP Clustering Filter Implements IP clustering as a default filter. --- lib/bridgedb/Dist.py | 64 ++++++++++++++++++++++++------------------------- 1 files changed, 31 insertions(+), 33 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index c151bbc..15d9653 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -42,8 +42,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): ## rings of this distributor. ## areaOrderHmac -- an hmac function used to order areas within rings. ## areaClusterHmac -- an hmac function used to assign areas to rings. - def __init__(self, areaMapper, nClusters, key, ipCategories=(), - answerParameters=None): + def __init__(self, areaMapper, nClusters, key, ipCategories=(), answerParameters=None): self.areaMapper = areaMapper self.answerParameters = answerParameters @@ -104,6 +103,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): be any string, so long as it changes with every period. N -- the number of bridges to try to give back. """ + if not bridgeFilterRules: bridgeFilterRules=[] logging.debug("getBridgesForIP(%s, %s, %s, %s" % (ip, epoch, N, bridgeFilterRules)) if not len(self.splitter): logging.debug("bailing without splitter") @@ -120,44 +120,42 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): pos = self.areaOrderHmac("category<%s>%s"%(epoch,area)) return ring.getBridges(pos, N, countryCode) - # dynamic filter construction - #XXX: may bypass clusters! front-end must supply clustering function - # or provide another mechanism (perhaps geoip based) to sort and select - # bridges from separate pools. - if bridgeFilterRules: - logging.debug("bridgeFilterRules: %s" % bridgeFilterRules) - #XXX: is there a better way to cache by ruleset signature? - ruleset = frozenset(bridgeFilterRules) - if ruleset in self.splitter.filterRings.keys(): - logging.debug("Cache hit %s" % ruleset) - _,ring = self.splitter.filterRings[ruleset] - else: - logging.debug("Cache miss %s" % ruleset) - # add new ring - #XXX what key do we use here? does it matter? - key1 = bridgedb.Bridges.get_hmac(self.splitter.key, str(bridgeFilterRules)) - ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - # debug log: cache miss - self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), - populate_from=self.splitter.bridges) - + # Bridge Filter Ruleset Construction + + # IP clustering; select only bridges from the corresponding cluster. + h = int( self.areaClusterHmac(area)[:8], 16) + # length of numClusters + clusterNum = h % len(self.rings) + + #XXX: assumes len(self.rings) = len(nClusters) + g = filterAssignBridgesToRing(self.splitter.hmac, + len(self.rings) + + len(self.categoryRings), + clusterNum) + bridgeFilterRules.append(g) + logging.debug("bridgeFilterRules: %s" % bridgeFilterRules) + #XXX: is there a better way to cache by ruleset signature? + ruleset = frozenset(bridgeFilterRules) + + # See if we have a cached copy of the ring, + # otherwise, add a new ring and populate it + if ruleset in self.splitter.filterRings.keys(): + logging.debug("Cache hit %s" % ruleset) + _,ring = self.splitter.filterRings[ruleset] else: - # Which bridge cluster should we look at? - h = int( self.areaClusterHmac(area)[:8], 16) - clusterNum = h % len(self.rings) - ring = self.rings[clusterNum] - # If a ring is empty, consider the next. - while not len(ring): - clusterNum = (clusterNum + 1) % len(self.rings) - ring = self.rings[clusterNum] + logging.debug("Cache miss %s" % ruleset) + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, + "Order-Bridges-In-Ring-%d"%clusterNum) + ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) + self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), + populate_from=self.splitter.bridges) # Now get the bridge. pos = self.areaOrderHmac("<%s>%s" % (epoch, area)) return ring.getBridges(pos, N) def __len__(self): - #XXX does not include ip categories or filtered rings - return sum(len(r) for r in self.rings) + return len(self.splitter) def dumpAssignments(self, f, description=""): self.splitter.dumpAssignments(f, description)

1 0

[bridgedb/master] 4297 - Update PortSpec parsing to reflect spec
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 94a5313837b30c07dd313e15715f1a66c7c39feb Author: aagbsn <aagbsn(a)extc.org> Date: Wed May 30 19:15:31 2012 -0700 4297 - Update PortSpec parsing to reflect spec proposal 186-multiple-orports.txt dropped port range support. --- lib/bridgedb/Bridges.py | 222 +++++++++++++++++------------------------------ lib/bridgedb/Server.py | 15 ++-- 2 files changed, 85 insertions(+), 152 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 281e5f2..1e15157 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -25,6 +25,7 @@ ID_LEN = 20 DIGESTMOD = sha HEX_DIGEST_LEN = 40 DIGEST_LEN = 20 +PORTSPEC_LEN = 16 def is_valid_ip(ip): """Return True if ip is the string encoding of a valid IPv4 address, @@ -65,17 +66,6 @@ def is_valid_fingerprint(fp): else: return True -def is_valid_or_address(or_address): - """Return true iff or_address is in the right format - (ip,frozenset(port)) or (ip, frozenset(port_low,port_high)) for ranges - """ - if len(or_address) != 2: return False - ip,port = or_address - if not is_valid_ip(ip): return False - if type(port) is not int: return False - if not (1 <= port <= 65535): return False - return True - toHex = binascii.b2a_hex fromHex = binascii.a2b_hex @@ -152,29 +142,31 @@ class Bridge: self.nickname, self.ip, self.orport, self.fingerprint) def getConfigLine(self,includeFingerprint=False, - selectFromORAddresses=False, needIPv4=True, needIPv6=False): """Return a line describing this bridge for inclusion in a torrc.""" # select an address:port from or-addresses - if selectFromORAddresses and self.or_addresses: - filtered_addresses = None - # bridges may have both classes. we only return one. - if needIPv6: - f = lambda x: type(x[0]) is ipaddr.IPv6Address - filtered_addresses = filter(f, self.or_addresses.items()) - elif needIPv4: - f = lambda x: type(x[0]) is ipaddr.IPv4Address - filtered_addresses = filter(f, self.or_addresses.items()) - if filtered_addresses: - address,portlist = random.choice(filtered_addresses) - if type(address) is ipaddr.IPv6Address: - ip = "[%s]"%address - else: - ip = "%s"%address - orport = portlist.getPort() #magic + filtered_addresses = None + # bridges may have both classes. we only return one. + if needIPv6: + f = lambda x: type(x[0]) is ipaddr.IPv6Address + filtered_addresses = filter(f, self.or_addresses.items()) + elif needIPv4: + f = lambda x: type(x[0]) is ipaddr.IPv4Address + filtered_addresses = filter(f, self.or_addresses.items()) + # default ip, orport should get a chance at being selected + if type(ipaddr.IPAddress(self.ip)) is ipaddr.IPv4Address: + filtered_addresses.append((ipaddr.IPAddress(self.ip), set([int(self.orport)]))) + + if filtered_addresses: + address,portlist = random.choice(filtered_addresses) + if type(address) is ipaddr.IPv6Address: + ip = "[%s]"%address + else: + ip = "%s"%address + orport = random.choice(list(portlist)) - # default to ip,orport ; ex. when logging + # default to ip,orport else: ip = self.ip orport = self.orport @@ -204,6 +196,12 @@ class Bridge: assert is_valid_ip(self.ip) assert is_valid_fingerprint(self.fingerprint) assert 1 <= self.orport <= 65535 + if self.or_addresses: + for address, portlist in self.or_addresses.items(): + assert is_valid_ip(address) + for port in portlist: + assert type(port) is int + assert 1 <= port <= 65535 def setStatus(self, running=None, stable=None): if running is not None: @@ -246,7 +244,7 @@ def parseDescFile(f, bridge_purpose='bridge'): IPV6ADDR = an ipv6 address, surrounded by square brackets. IPV4ADDR = an ipv4 address, represented as a dotted quad. PORTLIST = PORTSPEC | PORTSPEC "," PORTLIST - PORTSPEC = PORT | PORT "-" PORT + PORTSPEC = PORT PORT = a number between 1 and 65535 inclusive. """ @@ -273,10 +271,18 @@ def parseDescFile(f, bridge_purpose='bridge'): elif line.startswith("or-address "): if num_or_address_lines < 8: line = line[11:] - address,portlist = parseORAddressLine(line) try: - or_addresses[address].add(portlist) + address,portlist = parseORAddressLine(line) + except ParseORAddressError: + logging.warn("Invalid or-address line "\ + "from bridge with ID %r" %fingerprint) + continue + try: + # distinct ports only + portlist.add(or_addresses[address]) except KeyError: + pass + finally: or_addresses[address] = portlist else: logging.warn("Skipping extra or-address line "\ @@ -300,142 +306,70 @@ class PortList: def __init__(self, *args, **kwargs): self.ports = set() - self.ranges = [] - self.portdispenser = None - if len(args) == 1: - if type(args[0]) is str: - ports = [p.split('-') for p in args[0].split(',')] - # truncate per spec - ports = ports[:16] - for ps in ports: - try: ps = [int(x) for x in ps] - except ValueError: break - if len(ps) == 1: self.add(ps[0]) - elif len(ps) == 2: self.add(ps[0],ps[1]) - else: - self.add(args[0]) - elif len(args) == 2: - l,h = args - self.add(l,h) + self.add(*args) def _sanitycheck(self, val): #XXX: if debug=False this is disabled. bad! assert type(val) is int - assert(val > 0) - assert(val <= 65535) - - def __contains__(self, val1, val2=None): - self._sanitycheck(val1) - if val2: self.sanitycheck(val2) - - # check a single port - if not val2 and val1: - if val1 in self.ports: return True - for start,end in self.ranges: - f = lambda x: start <= x <= end - if f(val1): return True - return False - - if val2 and val1: - for start,end in self.ranges: - f = lambda x: start <= x <= end - if f(val1) and f(val2): return True - - for start,end in self.ranges: - f = lambda x: start <= x <= end - if f(val): return True - - def add(self, val1, val2=None): - self._sanitycheck(val1) - - # add as a single port instead - if val2 == val1: val2 = None - if val2: - self._sanitycheck(val2) - start = min(val1,val2) - end = max(val1,val2) - self.ranges.append((start,end)) - # reduce to largest continuous ranges - self._squash() - else: - if val1 in self: return - self.ports.add(val1) + assert(0 < val <= 65535) - # reset port dispenser - if self.portdispenser: - self.portdispenser = None - - def getPort(self): - # returns a single valid port - if not self.portdispenser: - self.portdispenser = self.__iter__() - try: - return self.portdispenser.next() - except StopIteration, AttributeError: - self.portdispenser = self.__iter__() - return self.portdispenser.next() - - def _squash(self): - # merge intersecting ranges - if len(self.ranges) > 1: - self.ranges.sort(key=lambda x: x[0]) - squashed = [self.ranges.pop(0)] - for r in self.ranges: - if (squashed[-1][0] <= r[0] <= squashed[-1][1]): - #intersection, extend r1, drop r2 - if r[1] > squashed[-1][1]: - squashed[-1] = (squashed[-1][0],r[1]) - # drop r - else: - # keep r - squashed.append(r) + def __contains__(self, val1): + return val1 in self.ports - self.ranges = squashed - - # drop enclosed ports - ports = self.ports.copy() - for p in self.ports: - for s,e in self.ranges: - if s <= p <= e: - ports.remove(p) - self.ports = ports + def add(self, *args): + for arg in args: + try: + if type(arg) is str: + ports = set([int(p) for p in arg.split(',')][:PORTSPEC_LEN]) + [self._sanitycheck(p) for p in ports] + self.ports.update(ports) + if type(arg) is int: + self._sanitycheck(arg) + self.ports.update([arg]) + if type(arg) is PortList: + self.add(list(arg.ports)) + except AssertionError: raise ValueError + except ValueError: raise def __iter__(self): - for p in self.ports: - yield p - for l,h in self.ranges: - # +1 for inclusive range - for rr in xrange(l,h+1): - yield rr + return self.ports.__iter__() def __str__(self): s = "" for p in self.ports: - s += "".join(", %s"%p) - for l,h in self.ranges: - s += ", %s-%s" % (l,h) - return s.lstrip(", ") + s += "".join(",%s"%p) + return s.lstrip(",") def __repr__(self): return "PortList('%s')" % self.__str__() -def parseORAddressLine(line): - #XXX should these go somewhere else? - re_ipv6 = re.compile("\[([a-fA-F0-9:]+)\]:(.*$)") - re_ipv4 = re.compile("((?:\d{1,3}\.?){4}):(.*$)") + def __len__(self): + return len(self.ports) +class ParseORAddressError(Exception): + def __init__(self): + msg = "Invalid or-address line" + Exception.__init__(self, msg) + +re_ipv6 = re.compile("\[([a-fA-F0-9:]+)\]:(.*$)") +re_ipv4 = re.compile("((?:\d{1,3}\.?){4}):(.*$)") + +def parseORAddressLine(line): address = None portlist = None # try regexp to discover ip version for regex in [re_ipv4, re_ipv6]: m = regex.match(line) if m: + # get an address and portspec, or raise ParseError try: address = ipaddr.IPAddress(m.group(1)) - portstring = m.group(2) - except IndexError, ValueError: break - portlist = PortList(portstring) - return address,portlist + portlist = PortList(m.group(2)) + except (IndexError, ValueError): raise ParseORAddressError + + # return a valid address, portlist or raise ParseORAddressError + if address and portlist and len(portlist): return address,portlist + raise ParseORAddressError def parseStatusFile(f): """DOCDOC""" @@ -806,7 +740,7 @@ class FilteredBridgeSplitter(BridgeHolder): def clear(self): #XXX syntax? [r.clear() for n,(f,r) in self.filterRings.items()] - self.bridges = [] + self.bridges = [] #self.filterRings = {} def insert(self, bridge): diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 039e9b7..5dfe888 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -156,7 +156,7 @@ class WebResource(twisted.web.resource.Resource): ipv6 = False if "ipv6" in request.postpath: ipv6 = True - rules = [] + rules = [] if ip: if ipv6: @@ -171,8 +171,7 @@ class WebResource(twisted.web.resource.Resource): if bridges: answer = "".join("%s %s\n" % ( - b.getConfigLine(self.includeFingerprints,needIPv6=ipv6, - selectFromORAddresses=ipv6), + b.getConfigLine(self.includeFingerprints,needIPv6=ipv6), (I18n.BRIDGEDB_TEXT[16] if b.isBlocked(countryCode) else "") ) for b in bridges) else: @@ -413,9 +412,9 @@ def getMailResponse(lines, ctx): try: interval = ctx.schedule.getInterval(time.time()) bridges = ctx.distributor.getBridgesForEmail(clientAddr, - interval, ctx.N, - countryCode=None, - bridgeFilterRules=bridgeFilterRules) + interval, ctx.N, + countryCode=None, + bridgeFilterRules=bridgeFilterRules) # Handle rate limited email except TooSoonEmail, e: logging.info("Got a mail too frequently; warning %r: %s.", @@ -465,8 +464,8 @@ def getMailResponse(lines, ctx): if bridges: with_fp = ctx.cfg.EMAIL_INCLUDE_FINGERPRINTS - answer = "".join(" %s\n" % b.getConfigLine(with_fp, needIPv6=ipv6,\ - selectFromORAddresses=ipv6) for b in bridges) + answer = "".join(" %s\n" %b.getConfigLine(with_fp, + needIPv6=ipv6) for b in bridges) else: answer = "(no bridges currently available)"

1 0

[bridgedb/master] 4297 - Implement ipCategories functionality
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit f022b905ca01a193aabd4d78107f27fce85c40cd Author: aagbsn <aagbsn(a)extc.org> Date: Fri May 25 16:53:06 2012 -0700 4297 - Implement ipCategories functionality Implement ipCategories with filters so that compound filters can also be used (e.g. filter for ipv6 bridges) --- lib/bridgedb/Dist.py | 93 +++++++++++++++++++++++------------------------- lib/bridgedb/Server.py | 10 ++--- 2 files changed, 49 insertions(+), 54 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 15d9653..995eafb 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -44,11 +44,14 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): ## areaClusterHmac -- an hmac function used to assign areas to rings. def __init__(self, areaMapper, nClusters, key, ipCategories=(), answerParameters=None): self.areaMapper = areaMapper + self.nClusters = nClusters self.answerParameters = answerParameters self.rings = [] - self.categoryRings = [] #DOCDDOC - self.categories = [] #DOCDOC + + self.categories = [] #DOCDOC + for c in ipCategories: + self.categories.append(c) key2 = bridgedb.Bridges.get_hmac(key, "Assign-Bridges-To-Rings") key3 = bridgedb.Bridges.get_hmac(key, "Order-Areas-In-Rings") @@ -58,36 +61,12 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): # add splitter and cache the default rings # plus leave room for dynamic filters - ring_cache_size = nClusters + len(ipCategories) + 5 + ring_cache_size = self.nClusters + len(ipCategories) + 5 self.splitter = bridgedb.Bridges.FilteredBridgeSplitter(key2, max_cached_rings=ring_cache_size) logging.debug("added splitter %s" % self.splitter) - # assign bridges using a filter function - for n in xrange(nClusters): - key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n) - ring = bridgedb.Bridges.BridgeRing(key1, answerParameters) - ring.setName("IP ring %s"%n) #XXX: should be n+1 for consistency? - - g = filterAssignBridgesToRing(self.splitter.hmac, - nClusters + len(ipCategories), n) - self.splitter.addRing(ring, ring.name, g) - self.rings.append(ring) - - n = nClusters - for c in ipCategories: - logging.info("Building ring: Order-Bridges-In-Ring-%d"%n) - key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n) - ring = bridgedb.Bridges.BridgeRing(key1, answerParameters) - ring.setName("IP category ring %s"%n) #XXX: should be n+1 for consistency? - self.categoryRings.append( ring ) - self.categories.append( (c, ring) ) - g = filterAssignBridgesToRing(self.splitter.hmac, - nClusters + len(ipCategories), n) - self.splitter.addRing(ring, ring.name, g) - n += 1 - def clear(self): self.splitter.clear() @@ -112,29 +91,48 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): area = self.areaMapper(ip) logging.info("area is %s" % area) - - for category, ring in self.categories: + + key1 = '' + pos = 0 + n = self.nClusters + + # only one of ip categories or area clustering is active + # try to match the request to an ip category + for category in self.categories: + # IP Categories logging.info("---------------------------------") if category.contains(ip): + g = Filters.filterAssignBridgesToRing(self.splitter.hmac, + self.nClusters + + len(self.categories), + n) + bridgeFilterRules.append(g) logging.info("category<%s>%s"%(epoch,area)) pos = self.areaOrderHmac("category<%s>%s"%(epoch,area)) - return ring.getBridges(pos, N, countryCode) - - # Bridge Filter Ruleset Construction + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, + "Order-Bridges-In-Ring-%d"%n) + break; + n += 1 - # IP clustering; select only bridges from the corresponding cluster. - h = int( self.areaClusterHmac(area)[:8], 16) - # length of numClusters - clusterNum = h % len(self.rings) + # if no category matches, use area clustering + else: + # IP clustering + h = int( self.areaClusterHmac(area)[:8], 16) + # length of numClusters + clusterNum = h % self.nClusters - #XXX: assumes len(self.rings) = len(nClusters) - g = filterAssignBridgesToRing(self.splitter.hmac, - len(self.rings) + - len(self.categoryRings), - clusterNum) - bridgeFilterRules.append(g) + g = Filters.filterAssignBridgesToRing(self.splitter.hmac, + self.nClusters + + len(self.categories), + clusterNum) + bridgeFilterRules.append(g) + pos = self.areaOrderHmac("<%s>%s" % (epoch, area)) + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, + "Order-Bridges-In-Ring-%d"%clusterNum) + logging.debug("bridgeFilterRules: %s" % bridgeFilterRules) - #XXX: is there a better way to cache by ruleset signature? + + # try to find a cached copy ruleset = frozenset(bridgeFilterRules) # See if we have a cached copy of the ring, @@ -142,16 +140,15 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): if ruleset in self.splitter.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) _,ring = self.splitter.filterRings[ruleset] + + # else create the ring and populate it else: logging.debug("Cache miss %s" % ruleset) - key1 = bridgedb.Bridges.get_hmac(self.splitter.key, - "Order-Bridges-In-Ring-%d"%clusterNum) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), + self.splitter.addRing(ring, ruleset, Filters.filterBridgesByRules(bridgeFilterRules), populate_from=self.splitter.bridges) - # Now get the bridge. - pos = self.areaOrderHmac("<%s>%s" % (epoch, area)) + # get the bridge. return ring.getBridges(pos, N) def __len__(self): diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 2de1289..4bdabb3 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -156,15 +156,13 @@ class WebResource(twisted.web.resource.Resource): ipv6 = False if "ipv6" in request.postpath: ipv6 = True + rules = [] + if ip: if ipv6: - #XXX: this ruleset bypasses areamapper and bridge clusters. - # for now, there are very few IPv6 bridges. - rules=[filterBridgesByIP6] + rules.append(filterBridgesByIP6) else: - #XXX: default to areamapper + cluster, does not use oraddress - rules = None - #rules=[filterBridgesByIP4] + rules.append(filterBridgesByIP4) bridges = self.distributor.getBridgesForIP(ip, interval, self.nBridgesToGive,

1 0

[bridgedb/master] 4297 - Adds EmailBasedDistributor filtering
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit ebd6b613e5ff0a1ecb850f7f1e887c895475255f Author: aagbsn <aagbsn(a)extc.org> Date: Fri May 25 17:16:04 2012 -0700 4297 - Adds EmailBasedDistributor filtering The EmailBasedDistributor can now be passed the same filters as the IPBasedDistributor. --- lib/bridgedb/Dist.py | 57 ++++++++++++++++++++--------------------------- lib/bridgedb/Main.py | 2 +- lib/bridgedb/Server.py | 12 ++++++---- 3 files changed, 32 insertions(+), 39 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 995eafb..00cea72 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -102,7 +102,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): # IP Categories logging.info("---------------------------------") if category.contains(ip): - g = Filters.filterAssignBridgesToRing(self.splitter.hmac, + g = filterAssignBridgesToRing(self.splitter.hmac, self.nClusters + len(self.categories), n) @@ -121,7 +121,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): # length of numClusters clusterNum = h % self.nClusters - g = Filters.filterAssignBridgesToRing(self.splitter.hmac, + g = filterAssignBridgesToRing(self.splitter.hmac, self.nClusters + len(self.categories), clusterNum) @@ -145,7 +145,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): else: logging.debug("Cache miss %s" % ruleset) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, ruleset, Filters.filterBridgesByRules(bridgeFilterRules), + self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), populate_from=self.splitter.bridges) # get the bridge. @@ -273,8 +273,6 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): self.emailHmac = bridgedb.Bridges.get_hmac_fn(key1, hex=False) key2 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring") - self.ring = bridgedb.Bridges.BridgeRing(key2, answerParameters) - self.ring.name = "email ring" # XXXX clear the store when the period rolls over! self.domainmap = domainmap self.domainrules = domainrules @@ -284,15 +282,9 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): self.splitter = bridgedb.Bridges.FilteredBridgeSplitter(key2, max_cached_rings=5) - #XXX default ring. - def g(x): - return True - - self.splitter.addRing(self.ring, self.ring.name, g) - def clear(self): self.splitter.clear() - self.ring.clear() # should be take care of by above + #self.ring.clear() # should be take care of by above def insert(self, bridge): """Assign a bridge to this distributor.""" @@ -306,6 +298,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): be any string, so long as it changes with every period. N -- the number of bridges to try to give back. """ + if not bridgeFilterRules: bridgeFilterRules=[] now = time.time() try: emailaddress = normalizeEmail(emailaddress, self.domainmap, @@ -340,34 +333,32 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): pos = self.emailHmac("<%s>%s" % (epoch, emailaddress)) ring = None - if bridgeFilterRules: - ruleset = frozenset(bridgeFilterRules) - if ruleset in self.splitter.filterRings.keys(): - logging.debug("Cache hit %s" % ruleset) - _,ring = self.splitter.filterRings[ruleset] - else: - # cache miss, add new ring - logging.debug("Cache miss %s" % ruleset) - # add new ring - #XXX what key do we use here? does it matter? - key1 = bridgedb.Bridges.get_hmac(self.splitter.key, - str(bridgeFilterRules)) - ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - # debug log: cache miss - self.splitter.addRing(ring, ruleset, - filterBridgesByRules(bridgeFilterRules), - populate_from=self.splitter.bridges) + ruleset = frozenset(bridgeFilterRules) + if ruleset in self.splitter.filterRings.keys(): + logging.debug("Cache hit %s" % ruleset) + _,ring = self.splitter.filterRings[ruleset] else: - ring = self.ring - - result = ring.getBridges(pos, N, countryCode) + # cache miss, add new ring + logging.debug("Cache miss %s" % ruleset) + + # add new ring + #XXX what key do we use here? does it matter? + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, + str(bridgeFilterRules)) + ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) + # debug log: cache miss + self.splitter.addRing(ring, ruleset, + filterBridgesByRules(bridgeFilterRules), + populate_from=self.splitter.bridges) + + result = ring.getBridges(pos, N) db.setEmailTime(emailaddress, now) db.commit() return result def __len__(self): - return len(self.ring) + return len(self.splitter) def cleanDatabase(self): db = bridgedb.Storage.getDB() diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 8ef6fc0..7d28684 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -319,7 +319,7 @@ def startup(cfg): proxyList.replaceProxyList(loadProxyList(cfg)) logging.info("%d bridges loaded", len(splitter)) if emailDistributor: - logging.info("%d for email", len(emailDistributor.ring)) + logging.info("%d for email", len(emailDistributor.splitter)) if ipDistributor: logging.info("%d for web:", len(ipDistributor.splitter)) logging.info(" by location set: %s", diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 4bdabb3..039e9b7 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -401,19 +401,21 @@ def getMailResponse(lines, ctx): # read subject, see if they want ipv6 ipv6 = False + bridgeFilterRules = [] for ln in lines: if "ipv6" in ln.strip().lower(): ipv6 = True - rules=[filterBridgesByIP6] + bridgeFilterRules.append(filterBridgesByIP6) + break else: - rules=[filterBridgesByIP4] + bridgeFilterRules.append(filterBridgesByIP4) try: interval = ctx.schedule.getInterval(time.time()) bridges = ctx.distributor.getBridgesForEmail(clientAddr, - interval, ctx.N, - countryCode=None, - bridgeFilterRules=rules) + interval, ctx.N, + countryCode=None, + bridgeFilterRules=bridgeFilterRules) # Handle rate limited email except TooSoonEmail, e: logging.info("Got a mail too frequently; warning %r: %s.",

1 0

[bridgedb/master] 5947 - Parse "a" lines from networkstatus
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 677272c11ea1a4b86fb143a12a76d03a6384b032 Author: aagbsn <aagbsn(a)extc.org> Date: Wed May 30 21:03:47 2012 -0700 5947 - Parse "a" lines from networkstatus The "a" lines have been tested by the Bridge Authority. The or-addresses in the bridge descriptors have not. This fix makes BridgeDB read or-addresses from the networkstatus instead of the bridge descriptors --- lib/bridgedb/Bridges.py | 46 +++++++++++++++++++++------------------------- lib/bridgedb/Main.py | 9 ++++++--- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 1e15157..9af5dfb 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -268,37 +268,14 @@ def parseDescFile(f, bridge_purpose='bridge'): orport = int(items[3]) elif line.startswith("fingerprint "): fingerprint = line[12:].replace(" ", "") - elif line.startswith("or-address "): - if num_or_address_lines < 8: - line = line[11:] - try: - address,portlist = parseORAddressLine(line) - except ParseORAddressError: - logging.warn("Invalid or-address line "\ - "from bridge with ID %r" %fingerprint) - continue - try: - # distinct ports only - portlist.add(or_addresses[address]) - except KeyError: - pass - finally: - or_addresses[address] = portlist - else: - logging.warn("Skipping extra or-address line "\ - "from Bridge with ID %r" % id) - num_or_address_lines += 1 elif line.startswith("router-signature"): purposeMatches = (purpose == bridge_purpose or bridge_purpose is None) if purposeMatches and nickname and ip and orport and fingerprint: - b = Bridge(nickname, ip, orport, fingerprint, - or_addresses=or_addresses) + b = Bridge(nickname, ip, orport, fingerprint) b.assertOK() yield b nickname = ip = orport = fingerprint = purpose = None - num_or_address_lines = 0 - or_addresses = {} class PortList: """ container class for port ranges @@ -374,6 +351,8 @@ def parseORAddressLine(line): def parseStatusFile(f): """DOCDOC""" ID = None + num_or_address_lines = 0 + or_addresses = {} for line in f: line = line.strip() if line.startswith("opt "): @@ -384,9 +363,26 @@ def parseStatusFile(f): ID = binascii.a2b_base64(line.split()[2]+"=") except binascii.Error: logging.warn("Unparseable base64 ID %r", line.split()[2]) + + elif ID and line.startswith("a "): + if num_or_address_lines < 8: + line = line[2:] + address,portlist = parseORAddressLine(line) + try: + or_addresses[address].add(portlist) + except KeyError: + or_addresses[address] = portlist + else: + logging.warn("Skipping extra or-address line "\ + "from Bridge with ID %r" % id) + num_or_address_lines += 1 + elif ID and line.startswith("s "): flags = line.split() - yield ID, ("Running" in flags), ("Stable" in flags) + yield ID, ("Running" in flags), ("Stable" in flags), or_addresses + ID = None + num_or_address_lines = 0 + or_addresses = {} def parseCountryBlockFile(f): """Generator. Parses a blocked-bridges file 'f', and yields a diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index fe34cb0..befb4ed 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -166,10 +166,12 @@ def load(cfg, splitter, clear=False): countryblock.clear() logging.info("Loading bridges") status = {} + addresses = {} if hasattr(cfg, "STATUS_FILE"): f = open(cfg.STATUS_FILE, 'r') - for ID, running, stable in Bridges.parseStatusFile(f): + for ID, running, stable, or_addresses in Bridges.parseStatusFile(f): status[ID] = running, stable + addresses[ID] = or_addresses f.close() if hasattr(cfg, "COUNTRY_BLOCK_FILE"): f = open(cfg.COUNTRY_BLOCK_FILE, 'r') @@ -183,6 +185,7 @@ def load(cfg, splitter, clear=False): if s is not None: running, stable = s bridge.setStatus(running=running, stable=stable) + bridge.or_addresses = addresses.get(bridge.getID()) bridge.setBlockingCountries( countryblock.getBlockingCountries(bridge.fingerprint)) splitter.insert(bridge) @@ -324,8 +327,8 @@ def startup(cfg): logging.info("%d for email", len(emailDistributor.splitter)) if ipDistributor: logging.info("%d for web:", len(ipDistributor.splitter)) - for (n,(f,r)) in ipDistributor.splitter.filterRings.items(): - logging.info(" by filter set %s, %d" % (n, len(r))) + for (n,(f,r)) in ipDistributor.splitter.filterRings.items(): + logging.info(" by filter set %s, %d" % (n, len(r))) #logging.info(" by location set: %s", # " ".join(str(len(r)) for r in ipDistributor.rings)) #logging.info(" by category set: %s",

1 0

[bridgedb/master] 4297 - Prepopulate ipCategories, clusters, and email rings
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit e6ce57e728802689544c130867edcadfeecd38ec Author: aagbsn <aagbsn(a)extc.org> Date: Tue May 29 16:01:12 2012 -0700 4297 - Prepopulate ipCategories, clusters, and email rings --- lib/bridgedb/Dist.py | 61 +++++++++++++++++++++++++++++++++++++++++++++----- lib/bridgedb/Main.py | 20 +++++++++------ 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 00cea72..ec13fb5 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -16,6 +16,8 @@ from ipaddr import IPv6Address, IPAddress from bridgedb.Filters import filterAssignBridgesToRing from bridgedb.Filters import filterBridgesByRules +from bridgedb.Filters import filterBridgesByIP4 +from bridgedb.Filters import filterBridgesByIP6 def uniformMap(ip): """Map an IP to an arbitrary 'area' string, such that any two /24 addresses @@ -67,6 +69,43 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): logging.debug("added splitter %s" % self.splitter) + def prepopulateRings(self): + # populate all rings (for dumping assignments and testing) + for filterFn in [None, filterBridgesByIP4, filterBridgesByIP6]: + n = self.nClusters + for category in self.categories: + g = filterAssignBridgesToRing(self.splitter.hmac, + self.nClusters + + len(self.categories), + n) + bridgeFilterRules = [g] + if filterFn: + bridgeFilterRules.append(filterFn) + ruleset = frozenset(bridgeFilterRules) + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, + "Order-Bridges-In-Ring-%d"%n) + n += 1 + ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) + self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), + populate_from=self.splitter.bridges) + + + # populate all ip clusters + for clusterNum in xrange(self.nClusters): + g = filterAssignBridgesToRing(self.splitter.hmac, + self.nClusters + + len(self.categories), + clusterNum) + bridgeFilterRules = [g] + if filterFn: + bridgeFilterRules.append(filterFn) + ruleset = frozenset(bridgeFilterRules) + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, + "Order-Bridges-In-Ring-%d"%clusterNum) + ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) + self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), + populate_from=self.splitter.bridges) + def clear(self): self.splitter.clear() @@ -100,7 +139,6 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): # try to match the request to an ip category for category in self.categories: # IP Categories - logging.info("---------------------------------") if category.contains(ip): g = filterAssignBridgesToRing(self.splitter.hmac, self.nClusters + @@ -290,15 +328,16 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): """Assign a bridge to this distributor.""" self.splitter.insert(bridge) - def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, countryCode=None, - bridgeFilterRules=None): + def getBridgesForEmail(self, emailaddress, epoch, N=1, + parameters=None, countryCode=None, bridgeFilterRules=None): """Return a list of bridges to give to a user. emailaddress -- the user's email address, as given in a from line. epoch -- the time period when we got this request. This can be any string, so long as it changes with every period. N -- the number of bridges to try to give back. """ - if not bridgeFilterRules: bridgeFilterRules=[] + if not bridgeFilterRules: + bridgeFilterRules=[] now = time.time() try: emailaddress = normalizeEmail(emailaddress, self.domainmap, @@ -342,9 +381,8 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): logging.debug("Cache miss %s" % ruleset) # add new ring - #XXX what key do we use here? does it matter? key1 = bridgedb.Bridges.get_hmac(self.splitter.key, - str(bridgeFilterRules)) + "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) # debug log: cache miss self.splitter.addRing(ring, ruleset, @@ -373,3 +411,14 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): def dumpAssignments(self, f, description=""): self.splitter.dumpAssignments(f, description) + + def prepopulateRings(self): + # populate all rings (for dumping assignments and testing) + for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: + ruleset = frozenset([filterFn]) + key1 = bridgedb.Bridges.get_hmac(self.splitter.key, + "Order-Bridges-In-Ring") + ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) + self.splitter.addRing(ring, ruleset, + filterBridgesByRules([filterFn]), + populate_from=self.splitter.bridges) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 7d28684..fe34cb0 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -285,6 +285,7 @@ def startup(cfg): Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"), categories, answerParameters=ringParams) + ipDistributor.prepopulateRings() # create default rings splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE) #webSchedule = Time.IntervalSchedule("day", 2) webSchedule = Time.NoSchedule() @@ -298,6 +299,7 @@ def startup(cfg): cfg.EMAIL_DOMAIN_MAP.copy(), cfg.EMAIL_DOMAIN_RULES.copy(), answerParameters=ringParams) + emailDistributor.prepopulateRings() # create default rings splitter.addRing(emailDistributor, "email", cfg.EMAIL_SHARE) #emailSchedule = Time.IntervalSchedule("day", 1) emailSchedule = Time.NoSchedule() @@ -322,14 +324,16 @@ def startup(cfg): logging.info("%d for email", len(emailDistributor.splitter)) if ipDistributor: logging.info("%d for web:", len(ipDistributor.splitter)) - logging.info(" by location set: %s", - " ".join(str(len(r)) for r in ipDistributor.rings)) - logging.info(" by category set: %s", - " ".join(str(len(r)) for r in ipDistributor.categoryRings)) - logging.info("Here are all known bridges in the category section:") - for r in ipDistributor.categoryRings: - for name, b in r.bridges.items(): - logging.info("%s" % b.getConfigLine(True)) + for (n,(f,r)) in ipDistributor.splitter.filterRings.items(): + logging.info(" by filter set %s, %d" % (n, len(r))) + #logging.info(" by location set: %s", + # " ".join(str(len(r)) for r in ipDistributor.rings)) + #logging.info(" by category set: %s", + # " ".join(str(len(r)) for r in ipDistributor.categoryRings)) + #logging.info("Here are all known bridges in the category section:") + #for r in ipDistributor.categoryRings: + # for name, b in r.bridges.items(): + # logging.info("%s" % b.getConfigLine(True)) # Dump bridge pool assignments to disk. try:

1 0

[bridgedb/master] 4297 - Update tests to reflect spec change
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 6bb42fca964b63fcf773bd60f38c8171236fb07c Author: aagbsn <aagbsn(a)extc.org> Date: Tue May 29 14:45:35 2012 -0700 4297 - Update tests to reflect spec change Adds a few more checks for parseORAddressLine() --- lib/bridgedb/Tests.py | 46 ++++++++++++++++++++++++++++++++-------------- 1 files changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 067eafa..59536ce 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -43,24 +43,16 @@ def randomPort(): def randomPortSpec(): """ - returns a random list of ports and port ranges (strings) - ranges are portlow-porthigh + returns a random list of ports """ ports = [] for i in range(0,24): ports.append(random.randint(1,65535)) ports.sort(reverse=True) - ports = [str(x) for x in ports] - - ranges = [] - for i in range(0,8): - ranges.append("%s-%s" % (ports.pop(),ports.pop())) - ports.extend(ranges) - portspec = "" for i in range(0,16): - portspec += "%s," % random.choice(ports) + portspec += "%d," % random.choice(ports) portspec = portspec.rstrip(',') #remove trailing , return portspec @@ -73,8 +65,14 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False): if or_addresses: for i in xrange(0,8): - b.or_addresses[ipaddr.IPv4Address(randomIP())] =\ - bridgedb.Bridges.PortList(randomPortSpec()) + address,portlist = bridgedb.Bridges.parseORAddressLine( + "%s:%s" % (randomIP(),randomPortSpec())) + try: + portlist.add(b.or_addresses[address]) + except KeyError: + pass + finally: + b.or_addresses[address] = portlist return b def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False): @@ -86,8 +84,14 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False): if or_addresses: for i in xrange(0,8): - b.or_addresses[ipaddr.IPv6Address(randomIP6())] =\ - bridgedb.Bridges.PortList(randomPortSpec()) + address,portlist = bridgedb.Bridges.parseORAddressLine( + "[%s]:%s" % (randomIP6(),randomPortSpec())) + try: + portlist.add(b.or_addresses[address]) + except KeyError: + pass + finally: + b.or_addresses[address] = portlist return b def fake16Bridge(orport=8080, running=True, stable=True): @@ -213,6 +217,9 @@ class IPBridgeDistTests(unittest.TestCase): for i in xrange(500): b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[filterBridgesByIP6]) + address, portlist = bridgedb.Bridges.parseORAddressLine( + random.choice(b).getConfigLine(needIPv6=True)[7:]) + assert type(address) is ipaddr.IPv6Address assert filterBridgesByIP6(random.choice(b)) def testDistWithFilterIP4(self): @@ -223,6 +230,10 @@ class IPBridgeDistTests(unittest.TestCase): for i in xrange(500): b = d.getBridgesForIP(randomIP(), "x", 1, bridgeFilterRules=[filterBridgesByIP4]) + address, portlist = bridgedb.Bridges.parseORAddressLine( + random.choice(b).getConfigLine(needIPv4=True)[7:]) + assert type(address) is ipaddr.IPv4Address + assert filterBridgesByIP4(random.choice(b)) def testDistWithFilterBoth(self): @@ -238,6 +249,13 @@ class IPBridgeDistTests(unittest.TestCase): t = b.pop() assert filterBridgesByIP4(t) assert filterBridgesByIP6(t) + address, portlist = bridgedb.Bridges.parseORAddressLine( + t.getConfigLine(needIPv4=True)[7:]) + assert type(address) is ipaddr.IPv4Address + address, portlist = bridgedb.Bridges.parseORAddressLine( + t.getConfigLine(needIPv6=True)[7:]) + assert type(address) is ipaddr.IPv6Address + def testDistWithFilterAll(self): d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo")

1 0

[bridgedb/master] 5948 - Do not pick from or-addresses randomly
by aagbsn＠torproject.org 16 Mar '13

16 Mar '13

commit 59c97a824be9ba8d09cff0f22f611ab3985f53a5 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 8 22:35:19 2012 -0700 5948 - Do not pick from or-addresses randomly Pick the same bridge line each time; uses hmac_fn and email or ip from the request. --- lib/bridgedb/Bridges.py | 68 +++++++++++++++++++++++++---------------------- 1 files changed, 36 insertions(+), 32 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 9af5dfb..4224fd7 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -141,40 +141,41 @@ class Bridge: return "Bridge(%r,%r,%d,%r)"%( self.nickname, self.ip, self.orport, self.fingerprint) - def getConfigLine(self,includeFingerprint=False, - needIPv4=True, needIPv6=False): - """Return a line describing this bridge for inclusion in a torrc.""" - - # select an address:port from or-addresses - filtered_addresses = None - # bridges may have both classes. we only return one. - if needIPv6: - f = lambda x: type(x[0]) is ipaddr.IPv6Address - filtered_addresses = filter(f, self.or_addresses.items()) - elif needIPv4: - f = lambda x: type(x[0]) is ipaddr.IPv4Address - filtered_addresses = filter(f, self.or_addresses.items()) - # default ip, orport should get a chance at being selected - if type(ipaddr.IPAddress(self.ip)) is ipaddr.IPv4Address: - filtered_addresses.append((ipaddr.IPAddress(self.ip), set([int(self.orport)]))) - - if filtered_addresses: - address,portlist = random.choice(filtered_addresses) - if type(address) is ipaddr.IPv6Address: - ip = "[%s]"%address + def getConfigLine(self, includeFingerprint=False, addressClass=None, + request=None): + """Returns a valid bridge line for inclusion in a torrc""" + #arguments: + # includeFingerprint + # addressClass - type of address to choose + # request - a string unique to this request + # e.g. email-address or uniformMap(ip) + + if not request: request = 'default' + digest = get_hmac_fn('Order-Or-Addresses')(request) + pos = long(digest[:8], 16) # lower 8 bytes -> long + + # default address type + if not addressClass: addressClass = ipaddr.IPv4Address + + # filter addresses by address class + addresses = filter(lambda x: isinstance(x[0], addressClass), + self.or_addresses.items()) + + # default ip, orport should get a chance at being selected + if isinstance(self.ip, addressClass): + addresses.insert(0,self.ip, PortList(self.orport)) + + if addresses: + address,portlist = addresses[pos % len(addresses)] + if isinstance(address, ipaddr.IPv6Address): ip = "[%s]"%address + else: ip = "%s"%address + orport = portlist[pos % len(portlist)] + + if includeFingerprint: + return "bridge %s:%d %s" % (ip, orport, self.fingerprint) else: - ip = "%s"%address - orport = random.choice(list(portlist)) + return "bridge %s:%d" % (ip, orport) - # default to ip,orport - else: - ip = self.ip - orport = self.orport - - if includeFingerprint: - return "bridge %s:%d %s" % (ip, orport, self.fingerprint) - else: - return "bridge %s:%d" % (ip, orport) def getAllConfigLines(self,includeFingerprint=False): """Generator. Iterate over all valid config lines for this bridge.""" @@ -323,6 +324,9 @@ class PortList: def __len__(self): return len(self.ports) + def __getitem__(self, x): + return list(self.ports)[x] + class ParseORAddressError(Exception): def __init__(self): msg = "Invalid or-address line"

1 0