April 2013 - tor-commits - lists.torproject.org

[bridgedb/master] 5482 - Add Tests for bridge stability code
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit 60357ba888e030cccfee0e003c48548738e7c1c0 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Oct 29 09:13:45 2012 -0700 5482 - Add Tests for bridge stability code --- lib/bridgedb/Tests.py | 94 ++++++++++++++++++++++++++++++++++++++++--------- 1 files changed, 77 insertions(+), 17 deletions(-) diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 3d1040d..383ad57 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -27,6 +27,8 @@ from bridgedb.Filters import filterBridgesByOnlyIP6 from bridgedb.Filters import filterBridgesByTransport from bridgedb.Filters import filterBridgesByNotBlockedIn +from bridgedb.Stability import BridgeHistory + def suppressWarnings(): warnings.filterwarnings('ignore', '.*tmpnam.*') @@ -158,7 +160,7 @@ simpleDesc = "router Unnamed %s %s 0 9030\n"\ "opt @purpose bridge\n" orAddress = "or-address %s:%s\n" def gettimestamp(): - ts = datetime.today().strftime("%Y-%M-%d %H:%m:%S") + ts = time.strftime("%Y-%M-%d %H:%m:%S") return "opt published %s\n" % ts class RhymesWith255Category: @@ -543,21 +545,6 @@ class SQLStorageTests(unittest.TestCase): db.cleanWarnedEmails(t+200) self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) - def testDescriptorStorage(self): - db = self.db - cur = self.cur - for i in xrange(10): - ts = [time.strftime("%Y-%M-%d %H:%m:%S") for x in xrange(3)] - b = random.choice([fakeBridge(), fakeBridge6()]) - [ db.addBridgeDescriptor(b.fingerprint, b.ip, b.orport, t) for t in ts ] - ds = db.getBridgeDescriptors(b.fingerprint) - assert(ds is not None) - for d in ds: - assert(d[3] in ts) - assert(d[0] == b.fingerprint) - assert(d[1] == b.ip) - assert(d[2] == b.orport) - class ParseDescFileTests(unittest.TestCase): def testSimpleDesc(self): test = "" @@ -638,12 +625,85 @@ class ParseDescFileTests(unittest.TestCase): #print "portlist: %s" % p #print "countries: %s" % c +class BridgeStabilityTests(unittest.TestCase): + def setUp(self): + self.fd, self.fname = tempfile.mkstemp() + self.db = bridgedb.Storage.Database(self.fname) + bridgedb.Storage.setGlobalDB(self.db) + self.cur = self.db._conn.cursor() + + def tearDown(self): + self.db.close() + os.close(self.fd) + os.unlink(self.fname) + + def testAddOrUpdateSingleBridgeHistory(self): + db = self.db + b = fakeBridge() + timestamp = time.time() + bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b, timestamp) + assert isinstance(bhe, BridgeHistory) + assert isinstance(db.getBridgeHistory(b.fingerprint), BridgeHistory) + assert len([y for y in db.getAllBridgeHistory()]) == 1 + + def testDeletingSingleBridgeHistory(self): + db = self.db + b = fakeBridge() + timestamp = time.time() + bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b, timestamp) + assert isinstance(bhe, BridgeHistory) + assert isinstance(db.getBridgeHistory(b.fingerprint), BridgeHistory) + db.delBridgeHistory(b.fingerprint) + assert db.getBridgeHistory(b.fingerprint) is None + assert len([y for y in db.getAllBridgeHistory()]) == 0 + + def testTOSA(self): + db = self.db + b = random.choice([fakeBridge,fakeBridge6])() + def timestampSeries(x): + for i in xrange(61): + yield i*60*30 + x # 30 minute intervals + now = time.time() + time_on_address = long(60*30*60) # 30 hours + downtime = 60*60*random.randint(0,4) # random hours of downtime + + for t in timestampSeries(now): + bridgedb.Stability.addOrUpdateBridgeHistory(b,t) + assert db.getBridgeHistory(b.fingerprint).tosa == time_on_address + + b.orport += 1 + + for t in timestampSeries(now + time_on_address + downtime): + bhe = bridgedb.Stability.addOrUpdateBridgeHistory(b,t) + assert db.getBridgeHistory(b.fingerprint).tosa == time_on_address + downtime + + def testLastSeenWithDifferentAddressAndPort(self): + db = self.db + for i in xrange(10): + ts = [] + ts = [ time.time() for x in xrange(5) ] + b = random.choice([fakeBridge(), fakeBridge6()]) + [ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ] + # last seen + last_seen = ts[-1] + # change the port + time_on_address = random.randint(60*60*.1,60*60*24*12) + ts = [ x + time_on_address for x in ts[:]] + b.orport = b.orport+1 + [ bridgedb.Stability.addOrUpdateBridgeHistory(b, t) for t in ts ] + b = db.getBridgeHistory(b.fingerprint) + assert b.tosa == time_on_address + assert (long(last_seen*1000) == b.lastSeenWithDifferentAddressAndPort) + assert (long(ts[-1]*1000) == b.lastSeenWithThisAddressAndPort) + + # test familiar + def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader() for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests, - EmailBridgeDistTests, ParseDescFileTests ]: + EmailBridgeDistTests, ParseDescFileTests, BridgeStabilityTests ]: suite.addTest(loader.loadTestsFromTestCase(klass)) for module in [ bridgedb.Bridges,

1 0

[bridgedb/master] 5482 - Add tests for descriptor storage methods
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit d29e647cac14ab20a7a5081be17c04dfcefbce41 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 23 07:51:16 2012 -0700 5482 - Add tests for descriptor storage methods --- lib/bridgedb/Tests.py | 56 +++++++++++++++++++++++++++++------------------- 1 files changed, 34 insertions(+), 22 deletions(-) diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index b4d3ca1..3d1040d 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -10,6 +10,7 @@ import tempfile import unittest import warnings import time +from datetime import datetime import bridgedb.Bridges import bridgedb.Main @@ -152,6 +153,14 @@ def fake16Bridge(orport=8080, running=True, stable=True): b.setStatus(running, stable) return b +simpleDesc = "router Unnamed %s %s 0 9030\n"\ +"opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ +"opt @purpose bridge\n" +orAddress = "or-address %s:%s\n" +def gettimestamp(): + ts = datetime.today().strftime("%Y-%M-%d %H:%m:%S") + return "opt published %s\n" % ts + class RhymesWith255Category: def contains(self, ip): return ip.endswith(".255") @@ -534,74 +543,76 @@ class SQLStorageTests(unittest.TestCase): db.cleanWarnedEmails(t+200) self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) + def testDescriptorStorage(self): + db = self.db + cur = self.cur + for i in xrange(10): + ts = [time.strftime("%Y-%M-%d %H:%m:%S") for x in xrange(3)] + b = random.choice([fakeBridge(), fakeBridge6()]) + [ db.addBridgeDescriptor(b.fingerprint, b.ip, b.orport, t) for t in ts ] + ds = db.getBridgeDescriptors(b.fingerprint) + assert(ds is not None) + for d in ds: + assert(d[3] in ts) + assert(d[0] == b.fingerprint) + assert(d[1] == b.ip) + assert(d[2] == b.orport) + class ParseDescFileTests(unittest.TestCase): def testSimpleDesc(self): - simpleDesc = "router Unnamed %s %s 0 9030\n"\ - "opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ - "opt @purpose bridge\n"\ - "router-signature\n" test = "" for i in range(100): test+= "".join(simpleDesc % (randomIP(), randomPort())) + test+=gettimestamp() + test+="router-signature\n" - bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: b.assertOK() def testSingleOrAddress(self): - simpleDesc = "router Unnamed %s %s 0 9030\n"\ - "opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ - "opt @purpose bridge\n" - orAddress = "or-address %s:%s\n" test = "" for i in range(100): test+= simpleDesc % (randomIP(), randomPort()) test+= orAddress % (randomIP(),randomPort()) + test+=gettimestamp() test+= "router-signature\n" - bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: b.assertOK() def testMultipleOrAddress(self): - simpleDesc = "router Unnamed %s %s 0 9030\n"\ - "opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ - "opt @purpose bridge\n" - orAddress = "or-address %s:%s\n" test = "" - for i in range(100): test+= simpleDesc % (randomIPString(), randomPort()) for i in xrange(8): test+= orAddress % (randomIPString(),randomPortSpec()) + test+=gettimestamp() test+= "router-signature\n" - bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: b.assertOK() def testConvolutedOrAddress(self): - simpleDesc = "router Unnamed %s %s 0 9030\n"\ - "opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ - "opt @purpose bridge\n" - orAddress = "or-address %s:%s\n" test = "" - for i in range(100): test+= simpleDesc % (randomIPString(), randomPort()) for i in xrange(8): test+= orAddress % (randomIPString(),randomPortSpec()) + test+=gettimestamp() test+= "router-signature\n" - bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: @@ -614,6 +625,7 @@ class ParseDescFileTests(unittest.TestCase): for i in range(100): test += simpleBlock % (randomIPString(), randomPort(), randomCountrySpec()) + test+=gettimestamp() for a,p,c in bridgedb.Bridges.parseCountryBlockFile(test.split('\n')): assert type(a) in (ipaddr.IPv6Address, ipaddr.IPv4Address)

1 0

[bridgedb/master] 5482 - add stability property convenience methods
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit 9bb52c00c1fb7ab7699b6b8b06acaf86588f0751 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 30 03:50:55 2012 -0700 5482 - add stability property convenience methods --- lib/bridgedb/Bridges.py | 46 +++++++++------------------------------------- 1 files changed, 9 insertions(+), 37 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index ed003ea..80dee5c 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -273,60 +273,32 @@ class Bridge: A bridge is 'familiar' if 1/8 of all active bridges have appeared more recently than it, or if it has been around for a Weighted Time of 8 days. """ - # if self.wt >= 8 days: return True - # bridges = session.query(Bridges).filter(time_first_seen, order=asc) - # if bridges[len(bridges) / 8].time_first_seen > self.time_first_seen: return True - # return False - pass + db = bridgedb.Storage.getDB() + return db.getBridgeHistory(self.wmtbac).familiar @property def wfu(self): """Weighted Fractional Uptime""" - pass + db = bridgedb.Storage.getDB() + return db.getBridgeHistory(self.wmtbac).weightedFractionalUptime @property def wt(self): """Weighted Time""" - pass + db = bridgedb.Storage.getDB() + return db.getBridgeHistory(self.wmtbac).weightedTime @property def wmtbac(self): """Weighted Mean Time Between Address Change""" - pass + db = bridgedb.Storage.getDB() + return db.getBridgeHistory(self.fingerprint).wmtbac @property def tosa(self): """the Time On Same Address (TOSA)""" - pass - - @property - def lastSeenWithDifferentAddressAndPort(self): - """Timestamp in milliseconds when this bridge was last - seen with a different address or port""" - db = bridgedb.Storage.getDB() - descs = db.getBridgeDescriptors(self.fingerprint) - assert(descs is not None) # should not happen - # sort by timestamp, newest first XXX: what is the db order? - descs.sort(lambda x,y: cmp(x[3] , y[3]), reverse=True) - # grab the most recent descriptor - last = descs.pop(0) - last_orport = last[2] # (fp, ip, orport, timestamp) - last_ip = last[1] - for desc in descs: - if desc[2] != last_orport or desc[1] != last_ip: - return desc[3]*1000 # timestamp in milliseconds - #XXX: hmm, we have never seen a different orport. What now? - return descs[-1][3]*1000 # the oldest descriptor known... maybe this is right - @property - def lastSeenWithThisAddressAndPort(self): - """Timestamp in milliseconds when this bridge was last - seen with this address and port""" - #XXX: what address and port would that be? db = bridgedb.Storage.getDB() - descs = db.getBridgeDescriptors(self.fingerprint) - assert(descs is not None) # should not happen - descs.sort(lambda x,y: cmp(x[3] , y[3]), reverse=True) - return descs.pop(0)[3]*1000 # just return the most recent timestamp + return db.getBridgeHistory(self.fingerprint).tosa def parseDescFile(f, bridge_purpose='bridge'): """Generator. Parses a cached-descriptors file 'f' and yeilds a Bridge object

1 0

[bridgedb/master] 5482 - Write TOSA into bucket file output
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit c74d953465257e391fdab09847530d8f9b017578 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 30 03:34:35 2012 -0700 5482 - Write TOSA into bucket file output --- lib/bridgedb/Bucket.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index 004633b..7f2a7e7 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -208,6 +208,8 @@ class BucketManager: f = open(filename, 'w') for b in bridges: line = "%s:%s" % (b.address, b.or_port) + bh = self.db.getBridgeHistory(b.hex_key) + if bh: line += "\n On address for %s" % bh.tosa f.write(line + '\n') f.close() except IOError:

1 0

[bridgedb/master] 5482 - Add BridgeHistory record class
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit afa94abf08fdb3989f8054859f2f239d060b10f8 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Oct 29 09:14:01 2012 -0700 5482 - Add BridgeHistory record class Adds a new table BridgeHistory and accompanying record class to help bridgedb track bridge stability over time. --- lib/bridgedb/Stability.py | 229 +++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 229 insertions(+), 0 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py new file mode 100644 index 0000000..0c2a288 --- /dev/null +++ b/lib/bridgedb/Stability.py @@ -0,0 +1,229 @@ +""" +This module provides functionality for tracking bridge stability metrics, using +the model introduced in [1] and implemented in [2]. + +[1] Karsten Loesing, An Analysis of Tor Bridge Stability. Technical Report. +The Tor Project, October 2011. +https://metrics.torproject.org/papers/bridge-stability-2011-10-31.pdf + +[2] https://gitweb.torproject.org/metrics-tasks/task-4255/SimulateBridgeStabili… +""" +import logging +import bridgedb.Storage + +# tunables +weighting_factor = long(19)/long(20) +discountIntervalMillis = long(60*60*12*1000) + +class BridgeHistory(object): + """ Record Class that tracks a single Bridge + The fields stored are: + + fingerprint, ip, port, weightedUptime, weightedTime, weightedRunLength, + totalRunWeights, lastSeenWithDifferentAddressAndPort, + lastSeenWithThisAddressAndPort, lastDiscountedHistoryValues. + + fingerprint The Bridge Fingerprint (unicode) + ip The Bridge IP (unicode) + port The Bridge orport (integer) + + weightedUptime Weighted uptime in seconds (long int) + weightedTime Weighted time in seconds (long int) + weightedRunLength Weighted run length of previous addresses or ports in + seconds. (long int) + totalRunWeights Total run weights of previously used addresses or + ports. (float) + + lastSeenWithDifferentAddressAndPort + Timestamp in milliseconds when this + bridge was last seen with a different address or port. (long int) + + lastSeenWithThisAddressAndPort + Timestamp in milliseconds when this bridge was last seen + with this address and port. (long int) + + lastDiscountedHistoryValues: + Timestamp in milliseconds when this bridge was last discounted. (long int) + """ + def __init__(self, fingerprint, ip, port, + weightedUptime, weightedTime, weightedRunLength, totalRunWeights, + lastSeenWithDifferentAddressAndPort, lastSeenWithThisAddressAndPort, + lastDiscountedHistoryValues=None): + self.fingerprint = fingerprint + self.ip = ip + self.port = port + self.weightedUptime = long(weightedUptime) + self.weightedTime = long(weightedTime) + self.weightedRunLength = float(weightedRunLength) + self.totalRunWeights = float(totalRunWeights) + self.lastSeenWithDifferentAddressAndPort = \ + long(lastSeenWithDifferentAddressAndPort) + self.lastSeenWithThisAddressAndPort = long(lastSeenWithThisAddressAndPort) + if lastDiscountedHistoryValues: + self.lastDiscountedHistoryValues = long(lastDiscountedHistoryValues) + else: + self.lastDiscountedHistoryValues = long(lastSeenWithThisAddressAndPort) + + def discountWeightedFractionalUptimeAndWeightedTime(self, discountUntilMillis): + """ discount weighted times """ + rounds = self.numDiscountRounds(discountUntilMillis) + if rounds > 0: + discount = lambda x: (weighting_factor**rounds)*x + self.weightedUptime = discount(self.weightedUptime) + self.weightedTime = discount(self.weightedTime) + self.weightedRunLength = discount(self.weightedRunLength) + self.totalRunWeights = discount(self.totalRunWeights) + + self.lastDiscountedHistoryValues += discountIntervalMillis * rounds + return rounds + + def numDiscountRounds(self, discountUntilMillis): + """ return the number of rounds of discounting needed to bring this + history element current """ + + result = discountUntilMillis - self.lastDiscountedHistoryValues + result = int(result/discountIntervalMillis) + return max(result,0) + + @property + def weightedFractionalUptime(self): + """Weighted Fractional Uptime""" + return long(10000) * self.weightedUptime / self.weightedTime + + @property + def tosa(self): + """the Time On Same Address (TOSA)""" + return ( self.lastSeenWithThisAddressAndPort - \ + self.lastSeenWithDifferentAddressAndPort ) / 1000 + + @property + def familiar(self): + """ + A bridge is 'familiar' if 1/8 of all active bridges have appeared + more recently than it, or if it has been around for a Weighted Time of 8 days. + """ + # if this bridge has been around longer than 8 days + if self.weightedTime >= long(8 * 24 * 60 * 60): + return True + + # return True if self.weightedTime is greater than the weightedTime + # of the > bottom 1/8 all bridges, sorted by weightedTime + db = bridgedb.Storage.getDB() + allWeightedTimes = [ bh.weightedTime for bh in db.getAllBridgeHistory()] + numBridges = len(allWeightedTimes) + logging.debug("Got %d weightedTimes", numBridges) + allWeightedTimes.sort(lambda x,y: cmp(x.weightedTime, y.weightedTime)) + if self.weighedTime > allWeighedTimes[numBridges/8].weightedTime: + return True + return False + + @property + def wmtbac(self): + """Weighted Mean Time Between Address Change""" + totalRunLength = self.weightedRunLength + \ + ((self.lastSeenWithThisAddressAndPort - + self.lastSeenWithDifferentAddressAndPort) / long(1000)) + + totalWeights = self.totalRunWeights + 1.0 + if totalWeights < 0.0001: return long(0) + assert(isinstance(long,totalRunLength)) + assert(isinstance(long,totalWeights)) + return totalRunlength / totalWeights + +def addOrUpdateBridgeHistory(bridge, timestamp): + db = bridgedb.Storage.getDB() + bhe = db.getBridgeHistory(bridge.fingerprint) + if not bhe: + # This is the first status, assume 60 minutes. + secondsSinceLastStatusPublication = long(60*60) + lastSeenWithDifferentAddressAndPort = timestamp * long(1000) + lastSeenWithThisAddressAndPort = timestamp * long(1000) + + bhe = BridgeHistory( + bridge.fingerprint, bridge.ip, bridge.orport, + 0,#weightedUptime + 0,#weightedTime + 0,#weightedRunLength + 0,# totalRunWeights + lastSeenWithDifferentAddressAndPort, # first timestamnp + lastSeenWithThisAddressAndPort + ) + + # Calculate the seconds since the last parsed status. If this is + # the first status or we haven't seen a status for more than 60 + # minutes, assume 60 minutes. + statusPublicationMillis = long(timestamp * 1000) + if (statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort) > 60*60*1000: + secondsSinceLastStatusPublication = long(60*60) + # otherwise, roll with it + else: + secondsSinceLastStatusPublication = \ + (statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort)/1000 + if statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort < 0: + # old descriptor, bail + logging.warn("Received old descriptor for bridge %s with timestamp %d", + bhe.fingerprint, statusPublicationMillis/1000) + return bhe + + # iterate over all known bridges and apply weighting factor + discountAndPruneBridgeHistories(statusPublicationMillis) + + # add or update the descriptor in history + db.updateIntoBridgeHistory(bhe) + bhe = None + + # Increment total weighted time for all bridges by seconds since the last + # status was published. Note, capped at 1 hour + for bh in db.getAllBridgeHistory(): + bh.weightedTime += secondsSinceLastStatusPublication + db.updateIntoBridgeHistory(bh) + + # For Running Bridges only: + # compare the stored history against the descriptor and see if the + # bridge has changed its address or port + bhe = db.getBridgeHistory(bridge.fingerprint) + if not bridge.running: + log.notice("%s is not running" % bridge.fingerprint) + return bhe + + # Parse the descriptor and see if the address or port changed + # If so, store the weighted run time + if bridge.orport != bhe.port or bridge.ip != bhe.ip: + bhe.totalRunWeights += 1.0; + bhe.weightedRunLength += bhe.tosa + bhe.lastSeenWithDifferentAddressAndPort =\ + bhe.lastSeenWithThisAddressAndPort + + # Regardless of whether the bridge is new, kept or changed + # its address and port, raise its WFU times and note its + # current address and port, and that we saw it using them. + bhe.weightedUptime += secondsSinceLastStatusPublication + bhe.lastSeenWithThisAddressAndPort = statusPublicationMillis + + bhe.ip = str(bridge.ip) + bhe.port = bridge.orport + return db.updateIntoBridgeHistory(bhe) + +def discountAndPruneBridgeHistories(discountUntilMillis): + db = bridgedb.Storage.getDB() + bhToRemove = [] + # Just check the first item to see if we're anywhere close yet + sample = None + for y in db.getAllBridgeHistory(): + sample = y + break + if not sample: return 0 + + if discountUntilMillis - sample.lastDiscountedHistoryValues < discountIntervalMillis: + return 0 + for bh in db.getAllBridgeHistory(): + # discount previous values by factor of 0.95 every 12 hours + bh.discountWeightedFractionalUptimeAndWeightedTime(discountUntilMillis) + + if (bh.weightedTime != 0 and (bh.weightedUptime * 10000 / bh.weightedTime) < 1): + bhToRemove.append(k) + else: + db.updateIntoBridgeHistory(bh) + + # prune items + for k in bhToRemove: db.delBridgeHistory(k)

1 0

[bridgedb/master] 5482 - Use info loglevel
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit f3ade3c6c5d057c93c6365400008b98bd4a9f908 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 30 07:22:09 2012 -0700 5482 - Use info loglevel --- lib/bridgedb/Storage.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 056dc4d..b9255d5 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -401,7 +401,8 @@ class Database: cur = self._cur cur.execute("SELECT * FROM BridgeHistory WHERE fingerprint = ?", (fp,)) h = cur.fetchone() - if h is None: return None + if h is None: + return return BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9]) def getAllBridgeHistory(self): @@ -447,7 +448,7 @@ def openDatabase(sqlite_file): cur.execute("SELECT value FROM Config WHERE key = 'schema-version'") val, = cur.fetchone() if val == 2: - logging.notice("Adding new table BridgeHistory") + logging.info("Adding new table BridgeHistory") cur.executescript(SCHEMA_2TO3_SCRIPT) elif val != 3: logging.warn("Unknown schema version %s in database.", val)

1 0

[bridgedb/master] 5482 - Annotate reserved bridge email by stability
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit 36c2c17b9d8bbdc37fd810d7f7f5ee058c51c9d7 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 30 05:01:51 2012 -0700 5482 - Annotate reserved bridge email by stability --- lib/bridgedb/Bucket.py | 18 ++++++++++++++---- 1 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index 7f2a7e7..bb8fa04 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -23,6 +23,7 @@ instead of 'unallocated'. This is why they are called pseudo-distributors. import time import bridgedb.Storage +from bridgedb.I18n import BRIDGEDB_TEXT # What should pseudo distributors be prefixed with in the database so we can # distinguish them from real distributors? @@ -204,12 +205,21 @@ class BucketManager: def dumpBridgesToFile(self, filename, bridges): """Dump a list of given bridges into a file """ + + + # get the bridge histories and sort by Time On Same Address + bridgeHistories = [] + for b in bridges: + bh = self.db.getBridgeHistory(b.hex_key) + if bh: bridgeHistories.append(bh) + bridgeHistories.sort(lambda x,y: cmp(x.tosa, y.tosa)) + try: f = open(filename, 'w') - for b in bridges: - line = "%s:%s" % (b.address, b.or_port) - bh = self.db.getBridgeHistory(b.hex_key) - if bh: line += "\n On address for %s" % bh.tosa + for bh in bridgeHistories: + days = bh.tosa / long(60*60*24) + line = "%s:%s\t(%d %s)" % \ + (bh.ip, bh.port, days, BRIDGEDB_TEXT[24]) f.write(line + '\n') f.close() except IOError:

1 0

[bridgedb/master] 5482 - Add stability string to I18n
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit 9a2081e02e32a6524a594ebd6919cbb21499b8f8 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 30 07:23:57 2012 -0700 5482 - Add stability string to I18n --- lib/bridgedb/I18n.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/lib/bridgedb/I18n.py b/lib/bridgedb/I18n.py index deb84cd..112b8ae 100644 --- a/lib/bridgedb/I18n.py +++ b/lib/bridgedb/I18n.py @@ -80,5 +80,7 @@ between emails is: """), # BRIDGEDB_TEXT[22] _("""Specify transport by name:"""), # BRIDGEDB_TEXT[23] - _("""Submit""") + _("""Submit"""), + # BRIDGEDB_TEXT[24] + _("""days at this address""") ]

1 0

[bridgedb/master] 5482 - Parse timestamp from ns descs.
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit cbfd761f10e0e9b2b194d11900f5c2a4ba525267 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Sep 6 15:51:23 2012 -0700 5482 - Parse timestamp from ns descs. Modifies parseDescFile to return an extra argument 'timestamp' --- lib/bridgedb/Bridges.py | 80 ++++++++++++++++++++++++++++++++++++++++++---- 1 files changed, 73 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 3117d34..ed003ea 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -266,6 +266,68 @@ class Bridge: except KeyError: return False # no blocklist return True + # Bridge Stability (#5482) properties. + @property + def familiar(self): + """ + A bridge is 'familiar' if 1/8 of all active bridges have appeared + more recently than it, or if it has been around for a Weighted Time of 8 days. + """ + # if self.wt >= 8 days: return True + # bridges = session.query(Bridges).filter(time_first_seen, order=asc) + # if bridges[len(bridges) / 8].time_first_seen > self.time_first_seen: return True + # return False + pass + + @property + def wfu(self): + """Weighted Fractional Uptime""" + pass + + @property + def wt(self): + """Weighted Time""" + pass + + @property + def wmtbac(self): + """Weighted Mean Time Between Address Change""" + pass + + @property + def tosa(self): + """the Time On Same Address (TOSA)""" + pass + + @property + def lastSeenWithDifferentAddressAndPort(self): + """Timestamp in milliseconds when this bridge was last + seen with a different address or port""" + db = bridgedb.Storage.getDB() + descs = db.getBridgeDescriptors(self.fingerprint) + assert(descs is not None) # should not happen + # sort by timestamp, newest first XXX: what is the db order? + descs.sort(lambda x,y: cmp(x[3] , y[3]), reverse=True) + # grab the most recent descriptor + last = descs.pop(0) + last_orport = last[2] # (fp, ip, orport, timestamp) + last_ip = last[1] + for desc in descs: + if desc[2] != last_orport or desc[1] != last_ip: + return desc[3]*1000 # timestamp in milliseconds + #XXX: hmm, we have never seen a different orport. What now? + return descs[-1][3]*1000 # the oldest descriptor known... maybe this is right + @property + def lastSeenWithThisAddressAndPort(self): + """Timestamp in milliseconds when this bridge was last + seen with this address and port""" + #XXX: what address and port would that be? + db = bridgedb.Storage.getDB() + descs = db.getBridgeDescriptors(self.fingerprint) + assert(descs is not None) # should not happen + descs.sort(lambda x,y: cmp(x[3] , y[3]), reverse=True) + return descs.pop(0)[3]*1000 # just return the most recent timestamp + def parseDescFile(f, bridge_purpose='bridge'): """Generator. Parses a cached-descriptors file 'f' and yeilds a Bridge object for every entry whose purpose matches bridge_purpose. @@ -295,7 +357,7 @@ def parseDescFile(f, bridge_purpose='bridge'): PORT = a number between 1 and 65535 inclusive. """ - nickname = ip = orport = fingerprint = purpose = None + nickname = ip = orport = fingerprint = purpose = timestamp = None num_or_address_lines = 0 or_addresses = {} @@ -303,7 +365,6 @@ def parseDescFile(f, bridge_purpose='bridge'): line = line.strip() if line.startswith("opt "): line = line[4:] - if line.startswith("@purpose "): items = line.split() purpose = items[1] @@ -315,14 +376,19 @@ def parseDescFile(f, bridge_purpose='bridge'): orport = int(items[3]) elif line.startswith("fingerprint "): fingerprint = line[12:].replace(" ", "") + elif line.startswith("published "): + if line.startswith("published "): + try: + timestamp = time.strptime(line[10:],"%Y-%M-%d %H:%m:%S") + timestamp = time.mktime(timestamp) + except ValueError: timestamp = None elif line.startswith("router-signature"): - purposeMatches = (purpose == bridge_purpose or - bridge_purpose is None) - if purposeMatches and nickname and ip and orport and fingerprint: + purposeMatches = (purpose == bridge_purpose or bridge_purpose is None) + if purposeMatches and nickname and ip and orport and fingerprint and timestamp: b = Bridge(nickname, ipaddr.IPAddress(ip), orport, fingerprint) b.assertOK() - yield b - nickname = ip = orport = fingerprint = purpose = None + yield b, timestamp + nickname = ip = orport = fingerprint = purpose = timestamp = None class PortList: """ container class for port ranges

1 0

[bridgedb/master] 5482 - Add database schema
by aagbsn＠torproject.org 16 Apr '13

16 Apr '13

commit 66ba955a564e06e91c471c15a44412d6ced46629 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Oct 21 06:29:39 2012 -0700 5482 - Add database schema Adds database schema, upgrade scripts, and helper methods for BridgeHistory objects --- lib/bridgedb/Storage.py | 90 +++++++++++++++++++++++++++++++++++++++++++++-- 1 files changed, 87 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 91c1a3c..056dc4d 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -9,6 +9,10 @@ import binascii import sqlite3 import time import sha +from ipaddr import IPAddress, IPv6Address, IPv4Address + +import bridgedb.Stability as Stability +from bridgedb.Stability import BridgeHistory toHex = binascii.b2a_hex fromHex = binascii.a2b_hex @@ -112,6 +116,8 @@ class SqliteDict: # Here is the SQL schema. + + SCHEMA2_SCRIPT = """ CREATE TABLE Config ( key PRIMARY KEY NOT NULL, @@ -155,6 +161,26 @@ SCHEMA2_SCRIPT = """ INSERT INTO Config VALUES ( 'schema-version', 2 ); """ +SCHEMA_2TO3_SCRIPT = """ + CREATE TABLE BridgeHistory ( + fingerprint PRIMARY KEY NOT NULL, + address, + port INT, + weightedUptime LONG, + weightedTime LONG, + weightedRunLength DOUBLE, + totalRunWeights DOUBLE, + lastSeenWithDifferentAddressAndPort LONG, + lastSeenWithThisAddressAndPort LONG, + lastDiscountedHistoryValues LONG + ); + + CREATE INDEX BridgeHistoryIndex on BridgeHistory ( fingerprint ); + + INSERT OR REPLACE INTO Config VALUES ( 'schema-version', 3 ); + """ +SCHEMA3_SCRIPT = SCHEMA2_SCRIPT + SCHEMA_2TO3_SCRIPT + class BridgeData: """Value class carrying bridge information: hex_key - The unique hex key of the given bridge @@ -358,6 +384,61 @@ class Database: cur.execute("DELETE FROM WarnedEmails WHERE when_warned < ?", (t,)) + def updateIntoBridgeHistory(self, bhe): + cur = self._cur + cur.execute("INSERT OR REPLACE INTO BridgeHistory values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (bhe.fingerprint, str(bhe.ip), bhe.port, + bhe.weightedUptime, bhe.weightedTime, bhe.weightedRunLength, + bhe.totalRunWeights, bhe.lastSeenWithDifferentAddressAndPort, + bhe.lastSeenWithThisAddressAndPort, bhe.lastDiscountedHistoryValues)) + return bhe + + def delBridgeHistory(self, fp): + cur = self._cur + cur.execute("DELETE FROM BridgeHistory WHERE fingerprint = ?", (fp,)) + + def getBridgeHistory(self, fp): + cur = self._cur + cur.execute("SELECT * FROM BridgeHistory WHERE fingerprint = ?", (fp,)) + h = cur.fetchone() + if h is None: return None + return BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9]) + + def getAllBridgeHistory(self): + cur = self._cur + v = cur.execute("SELECT * FROM BridgeHistory") + if v is None: return + for h in v: + yield BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9]) + + def addBridgeDescriptor(self, fp, ip, port, timestamp): + _ip = str(ip).strip('[]') + cur = self._cur + logging.debug("Adding Descriptor to BridgeDescriptors Table") + logging.debug("Values {0}, {1}, {2}, {3}".format(fp, ip, port, timestamp)) + cur.execute("INSERT OR REPLACE INTO BridgeDescriptors" + "(fingerprint,ip,orport,timestamp) VALUES (?,?,?,?)", + (fp, _ip, port, timestamp)) + + def getBridgeDescriptors(self, fp): + #XXX: should we limit to the last 28 days of descriptors? + cur = self._cur + cur.execute("SELECT fingerprint, ip, orport, timestamp FROM BridgeDescriptors WHERE fingerprint = ?", (fp,)) + v = cur.fetchall() + + # always return an iterable. + if v is None: return [] + + # return an IPv4Address or IPv6Address + return [(i[0], IPAddress(i[1]), i[2], i[3]) for i in v] + + def cleanBridgeDescriptors(self, timestamp=None): + cur = self._cur + # purge only descriptors older than timestamp + if timestamp: + cur.execute("DELETE * FROM BridgeDescriptors WHERE timestamp > ?", (timestamp,)) + else: cur.execute("DELETE * FROM BridgeDescriptors") + def openDatabase(sqlite_file): conn = sqlite3.Connection(sqlite_file) cur = conn.cursor() @@ -365,11 +446,14 @@ def openDatabase(sqlite_file): try: cur.execute("SELECT value FROM Config WHERE key = 'schema-version'") val, = cur.fetchone() - if val != 2: + if val == 2: + logging.notice("Adding new table BridgeHistory") + cur.executescript(SCHEMA_2TO3_SCRIPT) + elif val != 3: logging.warn("Unknown schema version %s in database.", val) except sqlite3.OperationalError: logging.warn("No Config table found in DB; creating tables") - cur.executescript(SCHEMA2_SCRIPT) + cur.executescript(SCHEMA3_SCRIPT) conn.commit() finally: cur.close() @@ -383,7 +467,7 @@ def openOrConvertDatabase(sqlite_file, db_file): conn = sqlite3.Connection(sqlite_file) cur = conn.cursor() - cur.executescript(SCHEMA2_SCRIPT) + cur.executescript(SCHEMA3_SCRIPT) conn.commit() import anydbm

1 0