April 2013 - tor-commits - lists.torproject.org

[bridgedb/master] 5482 - Fix timestamp parsing
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit c80488c43e0792b1b7fefab89dc5129bcbbdd63f Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 5 13:06:07 2012 -0800 5482 - Fix timestamp parsing --- lib/bridgedb/Bridges.py | 21 +++++++++++---------- 1 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 822385f..9bcc99a 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -283,7 +283,7 @@ class Bridge: return db.getBridgeHistory(self.fingerprint).weightedFractionalUptime @property - def wt(self): + def weightedTime(self): """Weighted Time""" db = bridgedb.Storage.getDB() return db.getBridgeHistory(self.fingerprint).weightedTime @@ -300,6 +300,12 @@ class Bridge: db = bridgedb.Storage.getDB() return db.getBridgeHistory(self.fingerprint).tosa + @property + def weightedUptime(self): + """Weighted Uptime""" + db = bridgedb.Storage.getDB() + return db.getBridgeHistory(self.fingerprint).weightedUptime + def parseDescFile(f, bridge_purpose='bridge'): """Generator. Parses a cached-descriptors file 'f' and yeilds a Bridge object for every entry whose purpose matches bridge_purpose. @@ -545,8 +551,9 @@ def parseStatusFile(f): if line.startswith("r "): try: ID = binascii.a2b_base64(line.split()[2]+"=") - timestamp = time.strptime(line.split()[4],"%Y-%m-%d %H:%M:%S") - timestamp = time.mktime(timestamp) + timestamp = time.mktime(time.strptime( + " ".join(line.split()[4:6]), "%Y-%m-%d %H:%M:%S") + ) except binascii.Error: logging.warn("Unparseable base64 ID %r", line.split()[2]) except ValueError: timestamp = None @@ -561,18 +568,12 @@ def parseStatusFile(f): or_addresses[address] = portlist else: logging.warn("Skipping extra or-address line "\ - "from Bridge with ID %r" % id) + "from Bridge with ID %r" % ID) num_or_address_lines += 1 elif ID and timestamp and line.startswith("s "): flags = line.split() yield ID, ("Running" in flags), ("Stable" in flags), or_addresses, timestamp - # add or update BridgeHistory entries into the database - # XXX: what do we do with all these or_addresses? - # The bridge stability metrics are only concerned with a single ip:port - # So for now, we will only consider the bridges primary IP:port - bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) - timestamp = ID = None num_or_address_lines = 0 or_addresses = {}

1 0

[bridgedb/master] 5482 - Keep track of timestamps parsed from NS
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit ef71dd383675af60399eed51da24884c8e75d54f Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 5 13:08:22 2012 -0800 5482 - Keep track of timestamps parsed from NS --- lib/bridgedb/Main.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 808da43..f081064 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -166,6 +166,7 @@ def load(cfg, splitter, clear=False): logging.info("Loading bridges") status = {} addresses = {} + timestamps = {} if hasattr(cfg, "STATUS_FILE"): f = open(cfg.STATUS_FILE, 'r') for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(f): @@ -191,7 +192,7 @@ def load(cfg, splitter, clear=False): # XXX: what do we do with all these or_addresses? # The bridge stability metrics are only concerned with a single ip:port # So for now, we will only consider the bridges primary IP:port - ts = timestamps[ID][:] + ts = timestamps[bridge.getID()][:] ts.sort() for timestamp in ts: bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp)

1 0

[bridgedb/master] 5482 - Do not update database inside query loop.
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 64230a4ecc6bc97932d70cddd0871d3d2108bab6 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 5 13:37:30 2012 -0800 5482 - Do not update database inside query loop. Fixes a bug where updated BridgeHistory objects are not stored in the db. --- lib/bridgedb/Stability.py | 21 +++++++++++++-------- 1 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index bdc4a4a..0f217a7 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -174,9 +174,12 @@ def addOrUpdateBridgeHistory(bridge, timestamp): # Increment total weighted time for all bridges by seconds since the last # status was published. Note, capped at 1 hour + toupdate = [] + for bh in db.getAllBridgeHistory(): bh.weightedTime += secondsSinceLastStatusPublication - db.updateIntoBridgeHistory(bh) + toupdate.append(bh) + [ db.updateIntoBridgeHistory(bh) for bh in toupdate ] # For Running Bridges only: # compare the stored history against the descriptor and see if the @@ -207,23 +210,25 @@ def addOrUpdateBridgeHistory(bridge, timestamp): def discountAndPruneBridgeHistories(discountUntilMillis): db = bridgedb.Storage.getDB() bhToRemove = [] + bhToUpdate = [] # Just check the first item to see if we're anywhere close yet sample = None for y in db.getAllBridgeHistory(): sample = y break - if not sample: return 0 + if not sample: return - if discountUntilMillis - sample.lastDiscountedHistoryValues < discountIntervalMillis: - return 0 + if discountUntilMillis - sample.lastDiscountedHistoryValues \ + < discountIntervalMillis: return for bh in db.getAllBridgeHistory(): # discount previous values by factor of 0.95 every 12 hours bh.discountWeightedFractionalUptimeAndWeightedTime(discountUntilMillis) - if (bh.weightedTime != 0 and (bh.weightedUptime * 10000 / bh.weightedTime) < 1): - bhToRemove.append(k) + if (bh.weightedTime != 0 and bh.weightedUptime != 0 and (bh.weightedUptime * 10000 / bh.weightedTime) < 1): + logging.debug("Removing bridge from history: %s" % bh.fingerprint) + bhToRemove.append(bh.fingerprint) else: - db.updateIntoBridgeHistory(bh) - + bhToUpdate.append(bh) + for k in bhToUpdate: db.updateIntoBridgeHistory(k) # prune items for k in bhToRemove: db.delBridgeHistory(k)

1 0

[bridgedb/master] 5482 - Fix familiar function
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 6639350579fa408849318fa1529fe940fa9003ec Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 5 13:34:21 2012 -0800 5482 - Fix familiar function Sort function is already operating on list of weightedTimes. --- lib/bridgedb/Stability.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index da66141..bdc4a4a 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -80,7 +80,6 @@ class BridgeHistory(object): def numDiscountRounds(self, discountUntilMillis): """ return the number of rounds of discounting needed to bring this history element current """ - result = discountUntilMillis - self.lastDiscountedHistoryValues result = int(result/discountIntervalMillis) return max(result,0) @@ -113,8 +112,8 @@ class BridgeHistory(object): allWeightedTimes = [ bh.weightedTime for bh in db.getAllBridgeHistory()] numBridges = len(allWeightedTimes) logging.debug("Got %d weightedTimes", numBridges) - allWeightedTimes.sort(lambda x,y: cmp(x.weightedTime, y.weightedTime)) - if self.weightedTime > allWeightedTimes[numBridges/8].weightedTime: + allWeightedTimes.sort() + if self.weightedTime >= allWeightedTimes[numBridges/8]: return True return False

1 0

[bridgedb/master] 5482 - Track when bridges were last discounted
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit e47be00f51ff8b56dee980e73a6c3006ce917312 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 12 09:36:32 2012 -0800 5482 - Track when bridges were last discounted Adds lastDiscountedHistoryValues to BridgeHistory records. --- lib/bridgedb/Stability.py | 22 ++++++++++++++-------- 1 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index 0f217a7..3b78fe3 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -44,28 +44,31 @@ class BridgeHistory(object): lastDiscountedHistoryValues: Timestamp in milliseconds when this bridge was last discounted. (long int) + + lastUpdatedWeightedTime: + Timestamp in milliseconds when the weighted time was updated. (long int) """ def __init__(self, fingerprint, ip, port, weightedUptime, weightedTime, weightedRunLength, totalRunWeights, lastSeenWithDifferentAddressAndPort, lastSeenWithThisAddressAndPort, - lastDiscountedHistoryValues=None): + lastDiscountedHistoryValues, lastUpdatedWeightedTime): self.fingerprint = fingerprint self.ip = ip self.port = port self.weightedUptime = long(weightedUptime) self.weightedTime = long(weightedTime) - self.weightedRunLength = float(weightedRunLength) + self.weightedRunLength = long(weightedRunLength) self.totalRunWeights = float(totalRunWeights) self.lastSeenWithDifferentAddressAndPort = \ long(lastSeenWithDifferentAddressAndPort) self.lastSeenWithThisAddressAndPort = long(lastSeenWithThisAddressAndPort) - if lastDiscountedHistoryValues: - self.lastDiscountedHistoryValues = long(lastDiscountedHistoryValues) - else: - self.lastDiscountedHistoryValues = long(lastSeenWithThisAddressAndPort) + self.lastDiscountedHistoryValues = long(lastDiscountedHistoryValues) + self.lastUpdatedWeightedTime = long(lastUpdatedWeightedTime) def discountWeightedFractionalUptimeAndWeightedTime(self, discountUntilMillis): """ discount weighted times """ + if self.lastDiscountedHistoryValues == 0: + self.lastDiscountedHistoryValues = discountUntilMillis rounds = self.numDiscountRounds(discountUntilMillis) if rounds > 0: discount = lambda x: (weighting_factor**rounds)*x @@ -146,9 +149,12 @@ def addOrUpdateBridgeHistory(bridge, timestamp): 0,#weightedRunLength 0,# totalRunWeights lastSeenWithDifferentAddressAndPort, # first timestamnp - lastSeenWithThisAddressAndPort + lastSeenWithThisAddressAndPort, + 0,#lastDiscountedHistoryValues, + 0,#lastUpdatedWeightedTime ) - + # first time we have seen this descriptor + db.updateIntoBridgeHistory(bhe) # Calculate the seconds since the last parsed status. If this is # the first status or we haven't seen a status for more than 60 # minutes, assume 60 minutes.

1 0

[bridgedb/master] 5482 - Refactor pruning code
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 8729f33c7538fdfc844b54f1d35ea65a0bef0040 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 12 09:43:23 2012 -0800 5482 - Refactor pruning code --- lib/bridgedb/Stability.py | 19 ++++++------------- 1 files changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index a8a8479..a86f853 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -161,6 +161,7 @@ def addOrUpdateBridgeHistory(bridge, timestamp): statusPublicationMillis = long(timestamp * 1000) if (statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort) > 60*60*1000: secondsSinceLastStatusPublication = long(60*60) + logging.debug("Capping secondsSinceLastStatusPublication to 1 hour") # otherwise, roll with it else: secondsSinceLastStatusPublication = \ @@ -181,6 +182,7 @@ def addOrUpdateBridgeHistory(bridge, timestamp): # compare the stored history against the descriptor and see if the # bridge has changed its address or port bhe = db.getBridgeHistory(bridge.fingerprint) + if not bridge.running: log.notice("%s is not running" % bridge.fingerprint) return bhe @@ -198,7 +200,6 @@ def addOrUpdateBridgeHistory(bridge, timestamp): # current address and port, and that we saw it using them. bhe.weightedUptime += secondsSinceLastStatusPublication bhe.lastSeenWithThisAddressAndPort = statusPublicationMillis - bhe.ip = str(bridge.ip) bhe.port = bridge.orport return db.updateIntoBridgeHistory(bhe) @@ -207,26 +208,18 @@ def discountAndPruneBridgeHistories(discountUntilMillis): db = bridgedb.Storage.getDB() bhToRemove = [] bhToUpdate = [] - # Just check the first item to see if we're anywhere close yet - sample = None - for y in db.getAllBridgeHistory(): - sample = y - break - if not sample: return - - if discountUntilMillis - sample.lastDiscountedHistoryValues \ - < discountIntervalMillis: return + for bh in db.getAllBridgeHistory(): # discount previous values by factor of 0.95 every 12 hours bh.discountWeightedFractionalUptimeAndWeightedTime(discountUntilMillis) - - if (bh.weightedTime != 0 and bh.weightedUptime != 0 and (bh.weightedUptime * 10000 / bh.weightedTime) < 1): + # give the thing at least 24 hours before pruning it + if bh.weightedFractionalUptime < 1 and bh.weightedTime > 60*60*24: logging.debug("Removing bridge from history: %s" % bh.fingerprint) bhToRemove.append(bh.fingerprint) else: bhToUpdate.append(bh) + for k in bhToUpdate: db.updateIntoBridgeHistory(k) - # prune items for k in bhToRemove: db.delBridgeHistory(k) def updateWeightedTime(statusPublicationMillis):

1 0

[bridgedb/master] 5482 - Refactor weighted time code
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 8a033f7ddd1937b9aaedcb9fd9f0b97a11339f09 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 12 09:39:51 2012 -0800 5482 - Refactor weighted time code --- lib/bridgedb/Stability.py | 30 +++++++++++++++++------------- 1 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index 3b78fe3..a8a8479 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -165,7 +165,7 @@ def addOrUpdateBridgeHistory(bridge, timestamp): else: secondsSinceLastStatusPublication = \ (statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort)/1000 - if statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort < 0: + if secondsSinceLastStatusPublication <= 0 and bhe.weightedTime > 0: # old descriptor, bail logging.warn("Received old descriptor for bridge %s with timestamp %d", bhe.fingerprint, statusPublicationMillis/1000) @@ -174,18 +174,8 @@ def addOrUpdateBridgeHistory(bridge, timestamp): # iterate over all known bridges and apply weighting factor discountAndPruneBridgeHistories(statusPublicationMillis) - # add or update the descriptor in history - db.updateIntoBridgeHistory(bhe) - bhe = None - - # Increment total weighted time for all bridges by seconds since the last - # status was published. Note, capped at 1 hour - toupdate = [] - - for bh in db.getAllBridgeHistory(): - bh.weightedTime += secondsSinceLastStatusPublication - toupdate.append(bh) - [ db.updateIntoBridgeHistory(bh) for bh in toupdate ] + # Update the weighted times of bridges + updateWeightedTime(statusPublicationMillis) # For Running Bridges only: # compare the stored history against the descriptor and see if the @@ -238,3 +228,17 @@ def discountAndPruneBridgeHistories(discountUntilMillis): for k in bhToUpdate: db.updateIntoBridgeHistory(k) # prune items for k in bhToRemove: db.delBridgeHistory(k) + +def updateWeightedTime(statusPublicationMillis): + bhToUpdate = [] + db = bridgedb.Storage.getDB() + for bh in db.getBridgesLastUpdatedBefore(statusPublicationMillis): + interval = (statusPublicationMillis - bh.lastUpdatedWeightedTime)/1000 + if interval > 0: + bh.weightedTime += min(3600,interval) # cap to 1hr + bh.lastUpdatedWeightedTime = statusPublicationMillis + #db.updateIntoBridgeHistory(bh) + bhToUpdate.append(bh) + + for bh in bhToUpdate: + db.updateIntoBridgeHistory(bh)

1 0

[bridgedb/master] 5482 - Add new fields to BridgeHistory table
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit b0edf1226753b7a6f78e5b921fc4fd22892b882f Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 12 09:44:47 2012 -0800 5482 - Add new fields to BridgeHistory table --- lib/bridgedb/Storage.py | 31 ++++++++++++++++++++----------- 1 files changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 8a07671..b0dada6 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -168,11 +168,12 @@ SCHEMA_2TO3_SCRIPT = """ port INT, weightedUptime LONG, weightedTime LONG, - weightedRunLength DOUBLE, + weightedRunLength LONG, totalRunWeights DOUBLE, lastSeenWithDifferentAddressAndPort LONG, lastSeenWithThisAddressAndPort LONG, - lastDiscountedHistoryValues LONG + lastDiscountedHistoryValues LONG, + lastUpdatedWeightedTime LONG ); CREATE INDEX BridgeHistoryIndex on BridgeHistory ( fingerprint ); @@ -384,14 +385,15 @@ class Database: cur.execute("DELETE FROM WarnedEmails WHERE when_warned < ?", (t,)) - def updateIntoBridgeHistory(self, bhe): + def updateIntoBridgeHistory(self, bh): cur = self._cur - cur.execute("INSERT OR REPLACE INTO BridgeHistory values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - (bhe.fingerprint, str(bhe.ip), bhe.port, - bhe.weightedUptime, bhe.weightedTime, bhe.weightedRunLength, - bhe.totalRunWeights, bhe.lastSeenWithDifferentAddressAndPort, - bhe.lastSeenWithThisAddressAndPort, bhe.lastDiscountedHistoryValues)) - return bhe + cur.execute("INSERT OR REPLACE INTO BridgeHistory values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (bh.fingerprint, str(bh.ip), bh.port, + bh.weightedUptime, bh.weightedTime, bh.weightedRunLength, + bh.totalRunWeights, bh.lastSeenWithDifferentAddressAndPort, + bh.lastSeenWithThisAddressAndPort, bh.lastDiscountedHistoryValues, + bh.lastUpdatedWeightedTime)) + return bh def delBridgeHistory(self, fp): cur = self._cur @@ -403,15 +405,22 @@ class Database: h = cur.fetchone() if h is None: return - return BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9]) + return BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9],h[10]) def getAllBridgeHistory(self): cur = self._cur v = cur.execute("SELECT * FROM BridgeHistory") if v is None: return + fp = {} for h in v: - yield BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9]) + yield BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9],h[10]) + def getBridgesLastUpdatedBefore(self, statusPublicationMillis): + cur = self._cur + v = cur.execute("SELECT * FROM BridgeHistory WHERE lastUpdatedWeightedTime < ?", (statusPublicationMillis,)) + if v is None: return + for h in v: + yield BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9],h[10]) def openDatabase(sqlite_file): conn = sqlite3.Connection(sqlite_file) cur = conn.cursor()

1 0

[bridgedb/master] 5482 - Add test for bridge familiarity
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit b5d633da7aee259bd70ec744c3f713f0c21cfa18 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 5 13:46:10 2012 -0800 5482 - Add test for bridge familiarity --- lib/bridgedb/Tests.py | 16 +++++++++++++++- 1 files changed, 15 insertions(+), 1 deletions(-) diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 79638f4..238f2c3 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -696,7 +696,21 @@ class BridgeStabilityTests(unittest.TestCase): assert (long(last_seen*1000) == b.lastSeenWithDifferentAddressAndPort) assert (long(ts[-1]*1000) == b.lastSeenWithThisAddressAndPort) - # test familiar + def testFamiliar(self): + # create some bridges + num_bridges = 100 + num_desc = 4*48 # 30m intervals, 48 per day + time_start = time.time() + bridges = [ fakeBridge() for x in xrange(num_bridges) ] + t = time.time() + ts = [ i*60*30+t for i in xrange(num_bridges) ] + for b in bridges: + time_series = [ 60*30*(i+1) + time_start for i in xrange(num_desc) ] + [ bridgedb.Stability.addOrUpdateBridgeHistory(b, i) for i in time_series ] + assert None not in bridges + # +1 to avoid rounding errors + assert bridges[-(num_bridges/8 + 1)].familiar == True + def testSuite(): suite = unittest.TestSuite()

1 0

[bridgedb/master] 3043 - Make make install actually install.
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit b4fdc57aa0789acd6bee55a3733ddc9177ec9934 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 19 15:45:28 2012 -0800 3043 - Make make install actually install. make install will just call python setup.py install. The pythonic way to install python packages with a setup.py is: python setup.py install, and now make install does just that. --- Makefile | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Makefile b/Makefile index bee8d75..e9abae4 100644 --- a/Makefile +++ b/Makefile @@ -6,4 +6,4 @@ test: python setup.py test install: - @echo 'To install, run python setup.py --prefix=$$HOME' + python setup.py install

1 0