April 2013 - tor-commits - lists.torproject.org

[bridgedb/master] 5482 - Sort bridges by weightedFractionalUptime
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 3a3940d19755ea7ce6939902025bdb21f27c0207 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 30 07:37:40 2012 -0700 5482 - Sort bridges by weightedFractionalUptime And print the time on same address next to the bridge line --- lib/bridgedb/Bucket.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index bb8fa04..6b13a7f 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -212,8 +212,8 @@ class BucketManager: for b in bridges: bh = self.db.getBridgeHistory(b.hex_key) if bh: bridgeHistories.append(bh) - bridgeHistories.sort(lambda x,y: cmp(x.tosa, y.tosa)) - + bridgeHistories.sort(lambda x,y: cmp(x.weightedFractionalUptime, + y.weightedFractionalUptime)) try: f = open(filename, 'w') for bh in bridgeHistories:

1 0

[bridgedb/master] 5482 - Do Not Divide by Zero
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 4f4b2ae119cfacda2c81e595313cb89657efa959 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 30 07:38:28 2012 -0700 5482 - Do Not Divide by Zero --- lib/bridgedb/Stability.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index 0c2a288..a54e67c 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -88,6 +88,7 @@ class BridgeHistory(object): @property def weightedFractionalUptime(self): """Weighted Fractional Uptime""" + if self.weightedTime <0.0001: return long(0) return long(10000) * self.weightedUptime / self.weightedTime @property

1 0

[bridgedb/master] 5482 - Add reachability information to bucket text
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit b7b3a6c57c77b1b81281f26ff1be25fcb05a770d Author: aagbsn <aagbsn(a)extc.org> Date: Tue Oct 30 08:37:18 2012 -0700 5482 - Add reachability information to bucket text --- lib/bridgedb/Bucket.py | 18 ++++++++++++++++-- 1 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index 6b13a7f..2554bed 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -24,6 +24,10 @@ instead of 'unallocated'. This is why they are called pseudo-distributors. import time import bridgedb.Storage from bridgedb.I18n import BRIDGEDB_TEXT +import bridgedb.Bridges +import binascii +toHex = binascii.b2a_hex + # What should pseudo distributors be prefixed with in the database so we can # distinguish them from real distributors? @@ -205,8 +209,6 @@ class BucketManager: def dumpBridgesToFile(self, filename, bridges): """Dump a list of given bridges into a file """ - - # get the bridge histories and sort by Time On Same Address bridgeHistories = [] for b in bridges: @@ -214,12 +216,24 @@ class BucketManager: if bh: bridgeHistories.append(bh) bridgeHistories.sort(lambda x,y: cmp(x.weightedFractionalUptime, y.weightedFractionalUptime)) + + # for a bridge, get the list of countries it might not work in + blocklist = dict() + if hasattr(self.cfg, "COUNTRY_BLOCK_FILE"): + f = open(self.cfg.COUNTRY_BLOCK_FILE, 'r') + for ID,address,portlist,countries in bridgedb.Bridges.parseCountryBlockFile(f): + blocklist[toHex(ID)] = countries + f.close() + try: f = open(filename, 'w') for bh in bridgeHistories: days = bh.tosa / long(60*60*24) line = "%s:%s\t(%d %s)" % \ (bh.ip, bh.port, days, BRIDGEDB_TEXT[24]) + if str(bh.fingerprint) in blocklist.keys(): + line = line + "\t%s: (%s)" % (BRIDGEDB_TEXT[16], + ",".join(blocklist[bh.fingerprint]),) f.write(line + '\n') f.close() except IOError:

1 0

[bridgedb/master] 5482 - Do not use integer division
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 08f7c3434723518156344372008d5074f78de5cc Author: aagbsn <aagbsn(a)extc.org> Date: Sun Nov 4 07:24:54 2012 -0800 5482 - Do not use integer division These should be floats, not longs --- lib/bridgedb/Stability.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index a4abcd9..da66141 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -12,7 +12,7 @@ import logging import bridgedb.Storage # tunables -weighting_factor = long(19)/long(20) +weighting_factor = float(19)/float(20) discountIntervalMillis = long(60*60*12*1000) class BridgeHistory(object):

1 0

[bridgedb/master] 5482 - Pass fingerprint to getBridgeHistory, not wmtbac
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 4423cc14b58a9efbac957c02c572a41982b8308b Author: aagbsn <aagbsn(a)extc.org> Date: Sun Nov 4 07:16:27 2012 -0800 5482 - Pass fingerprint to getBridgeHistory, not wmtbac --- lib/bridgedb/Bridges.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 80dee5c..d67d621 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -274,19 +274,19 @@ class Bridge: more recently than it, or if it has been around for a Weighted Time of 8 days. """ db = bridgedb.Storage.getDB() - return db.getBridgeHistory(self.wmtbac).familiar + return db.getBridgeHistory(self.fingerprint).familiar @property def wfu(self): """Weighted Fractional Uptime""" db = bridgedb.Storage.getDB() - return db.getBridgeHistory(self.wmtbac).weightedFractionalUptime + return db.getBridgeHistory(self.fingerprint).weightedFractionalUptime @property def wt(self): """Weighted Time""" db = bridgedb.Storage.getDB() - return db.getBridgeHistory(self.wmtbac).weightedTime + return db.getBridgeHistory(self.fingerprint).weightedTime @property def wmtbac(self):

1 0

[bridgedb/master] 5482 - fix typo in variable names
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 680ad65df00d16ee826be5a94a298c3f6a5a22a4 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Nov 4 07:23:25 2012 -0800 5482 - fix typo in variable names weighedTime and allWeighedTime should be: weightedTime and allWeightedTime --- lib/bridgedb/Stability.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/lib/bridgedb/Stability.py b/lib/bridgedb/Stability.py index a54e67c..a4abcd9 100644 --- a/lib/bridgedb/Stability.py +++ b/lib/bridgedb/Stability.py @@ -114,7 +114,7 @@ class BridgeHistory(object): numBridges = len(allWeightedTimes) logging.debug("Got %d weightedTimes", numBridges) allWeightedTimes.sort(lambda x,y: cmp(x.weightedTime, y.weightedTime)) - if self.weighedTime > allWeighedTimes[numBridges/8].weightedTime: + if self.weightedTime > allWeightedTimes[numBridges/8].weightedTime: return True return False

1 0

[bridgedb/master] 5482 - Fix timestamp formatting
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 23c708ac336a1e171d30ea904343aec9bd13d4da Author: aagbsn <aagbsn(a)extc.org> Date: Sun Nov 4 07:21:03 2012 -0800 5482 - Fix timestamp formatting %M and %m should be reversed. --- lib/bridgedb/Bridges.py | 2 +- lib/bridgedb/Tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index d67d621..223a78d 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -351,7 +351,7 @@ def parseDescFile(f, bridge_purpose='bridge'): elif line.startswith("published "): if line.startswith("published "): try: - timestamp = time.strptime(line[10:],"%Y-%M-%d %H:%m:%S") + timestamp = time.strptime(line[10:],"%Y-%m-%d %H:%M:%S") timestamp = time.mktime(timestamp) except ValueError: timestamp = None elif line.startswith("router-signature"): diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 383ad57..505a7b7 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -160,7 +160,7 @@ simpleDesc = "router Unnamed %s %s 0 9030\n"\ "opt @purpose bridge\n" orAddress = "or-address %s:%s\n" def gettimestamp(): - ts = time.strftime("%Y-%M-%d %H:%m:%S") + ts = time.strftime("%Y-%m-%d %H:%M:%S") return "opt published %s\n" % ts class RhymesWith255Category:

1 0

[bridgedb/master] 5482 - Remove references to bridge descriptors
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit b306d2d16b9a37245618f6c25ed10cce64c401f1 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Nov 4 07:26:39 2012 -0800 5482 - Remove references to bridge descriptors Removes unused code --- lib/bridgedb/Storage.py | 28 ---------------------------- 1 files changed, 0 insertions(+), 28 deletions(-) diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index b9255d5..8a07671 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -412,34 +412,6 @@ class Database: for h in v: yield BridgeHistory(h[0],IPAddress(h[1]),h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9]) - def addBridgeDescriptor(self, fp, ip, port, timestamp): - _ip = str(ip).strip('[]') - cur = self._cur - logging.debug("Adding Descriptor to BridgeDescriptors Table") - logging.debug("Values {0}, {1}, {2}, {3}".format(fp, ip, port, timestamp)) - cur.execute("INSERT OR REPLACE INTO BridgeDescriptors" - "(fingerprint,ip,orport,timestamp) VALUES (?,?,?,?)", - (fp, _ip, port, timestamp)) - - def getBridgeDescriptors(self, fp): - #XXX: should we limit to the last 28 days of descriptors? - cur = self._cur - cur.execute("SELECT fingerprint, ip, orport, timestamp FROM BridgeDescriptors WHERE fingerprint = ?", (fp,)) - v = cur.fetchall() - - # always return an iterable. - if v is None: return [] - - # return an IPv4Address or IPv6Address - return [(i[0], IPAddress(i[1]), i[2], i[3]) for i in v] - - def cleanBridgeDescriptors(self, timestamp=None): - cur = self._cur - # purge only descriptors older than timestamp - if timestamp: - cur.execute("DELETE * FROM BridgeDescriptors WHERE timestamp > ?", (timestamp,)) - else: cur.execute("DELETE * FROM BridgeDescriptors") - def openDatabase(sqlite_file): conn = sqlite3.Connection(sqlite_file) cur = conn.cursor()

1 0

[bridgedb/master] 5482 - Parse timestamp from networkstatus
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit 59bcda62f86ffbcc2b2c75e42a3a12d1150e1dae Author: aagbsn <aagbsn(a)extc.org> Date: Sun Nov 4 07:47:02 2012 -0800 5482 - Parse timestamp from networkstatus Bridge descriptors are only updated every 12-18 hours, but ns is updated every 30 minutes. --- lib/bridgedb/Bridges.py | 31 +++++++++++++++++-------------- lib/bridgedb/Main.py | 11 ++++++++--- lib/bridgedb/Tests.py | 8 ++++---- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 223a78d..822385f 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -329,7 +329,7 @@ def parseDescFile(f, bridge_purpose='bridge'): PORT = a number between 1 and 65535 inclusive. """ - nickname = ip = orport = fingerprint = purpose = timestamp = None + nickname = ip = orport = fingerprint = purpose = None num_or_address_lines = 0 or_addresses = {} @@ -348,19 +348,13 @@ def parseDescFile(f, bridge_purpose='bridge'): orport = int(items[3]) elif line.startswith("fingerprint "): fingerprint = line[12:].replace(" ", "") - elif line.startswith("published "): - if line.startswith("published "): - try: - timestamp = time.strptime(line[10:],"%Y-%m-%d %H:%M:%S") - timestamp = time.mktime(timestamp) - except ValueError: timestamp = None elif line.startswith("router-signature"): purposeMatches = (purpose == bridge_purpose or bridge_purpose is None) - if purposeMatches and nickname and ip and orport and fingerprint and timestamp: + if purposeMatches and nickname and ip and orport and fingerprint: b = Bridge(nickname, ipaddr.IPAddress(ip), orport, fingerprint) b.assertOK() - yield b, timestamp - nickname = ip = orport = fingerprint = purpose = timestamp = None + yield b + nickname = ip = orport = fingerprint = purpose = None class PortList: """ container class for port ranges @@ -540,7 +534,7 @@ def parseExtraInfoFile(f): def parseStatusFile(f): """DOCDOC""" - ID = None + timestamp = ID = None num_or_address_lines = 0 or_addresses = {} for line in f: @@ -551,8 +545,11 @@ def parseStatusFile(f): if line.startswith("r "): try: ID = binascii.a2b_base64(line.split()[2]+"=") + timestamp = time.strptime(line.split()[4],"%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timestamp) except binascii.Error: logging.warn("Unparseable base64 ID %r", line.split()[2]) + except ValueError: timestamp = None elif ID and line.startswith("a "): if num_or_address_lines < 8: @@ -567,10 +564,16 @@ def parseStatusFile(f): "from Bridge with ID %r" % id) num_or_address_lines += 1 - elif ID and line.startswith("s "): + elif ID and timestamp and line.startswith("s "): flags = line.split() - yield ID, ("Running" in flags), ("Stable" in flags), or_addresses - ID = None + yield ID, ("Running" in flags), ("Stable" in flags), or_addresses, timestamp + # add or update BridgeHistory entries into the database + # XXX: what do we do with all these or_addresses? + # The bridge stability metrics are only concerned with a single ip:port + # So for now, we will only consider the bridges primary IP:port + bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) + + timestamp = ID = None num_or_address_lines = 0 or_addresses = {} diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 56a52dd..808da43 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -168,16 +168,18 @@ def load(cfg, splitter, clear=False): addresses = {} if hasattr(cfg, "STATUS_FILE"): f = open(cfg.STATUS_FILE, 'r') - for ID, running, stable, or_addresses in Bridges.parseStatusFile(f): + for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(f): status[ID] = running, stable addresses[ID] = or_addresses + if ID in timestamps.keys(): timestamps[ID].append(timestamp) + else: timestamps[ID] = [timestamp] #transports[ID] = transports f.close() bridges = {} db = bridgedb.Storage.getDB() for fname in cfg.BRIDGE_FILES: f = open(fname, 'r') - for bridge, timestamp in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): + for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): bridges[bridge.getID()] = bridge s = status.get(bridge.getID()) if s is not None: @@ -189,7 +191,10 @@ def load(cfg, splitter, clear=False): # XXX: what do we do with all these or_addresses? # The bridge stability metrics are only concerned with a single ip:port # So for now, we will only consider the bridges primary IP:port - bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) + ts = timestamps[ID][:] + ts.sort() + for timestamp in ts: + bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) f.close() # read pluggable transports from extra-info document # XXX: should read from networkstatus after bridge-authority diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 505a7b7..79638f4 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -554,7 +554,7 @@ class ParseDescFileTests(unittest.TestCase): test+=gettimestamp() test+="router-signature\n" - bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: @@ -569,7 +569,7 @@ class ParseDescFileTests(unittest.TestCase): test+=gettimestamp() test+= "router-signature\n" - bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: @@ -584,7 +584,7 @@ class ParseDescFileTests(unittest.TestCase): test+=gettimestamp() test+= "router-signature\n" - bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: @@ -599,7 +599,7 @@ class ParseDescFileTests(unittest.TestCase): test+=gettimestamp() test+= "router-signature\n" - bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs:

1 0

[bridgedb/master] 5482 - Fix timestamp parsing
by aagbsn＠torproject.org 17 Apr '13

17 Apr '13

commit c80488c43e0792b1b7fefab89dc5129bcbbdd63f Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 5 13:06:07 2012 -0800 5482 - Fix timestamp parsing --- lib/bridgedb/Bridges.py | 21 +++++++++++---------- 1 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 822385f..9bcc99a 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -283,7 +283,7 @@ class Bridge: return db.getBridgeHistory(self.fingerprint).weightedFractionalUptime @property - def wt(self): + def weightedTime(self): """Weighted Time""" db = bridgedb.Storage.getDB() return db.getBridgeHistory(self.fingerprint).weightedTime @@ -300,6 +300,12 @@ class Bridge: db = bridgedb.Storage.getDB() return db.getBridgeHistory(self.fingerprint).tosa + @property + def weightedUptime(self): + """Weighted Uptime""" + db = bridgedb.Storage.getDB() + return db.getBridgeHistory(self.fingerprint).weightedUptime + def parseDescFile(f, bridge_purpose='bridge'): """Generator. Parses a cached-descriptors file 'f' and yeilds a Bridge object for every entry whose purpose matches bridge_purpose. @@ -545,8 +551,9 @@ def parseStatusFile(f): if line.startswith("r "): try: ID = binascii.a2b_base64(line.split()[2]+"=") - timestamp = time.strptime(line.split()[4],"%Y-%m-%d %H:%M:%S") - timestamp = time.mktime(timestamp) + timestamp = time.mktime(time.strptime( + " ".join(line.split()[4:6]), "%Y-%m-%d %H:%M:%S") + ) except binascii.Error: logging.warn("Unparseable base64 ID %r", line.split()[2]) except ValueError: timestamp = None @@ -561,18 +568,12 @@ def parseStatusFile(f): or_addresses[address] = portlist else: logging.warn("Skipping extra or-address line "\ - "from Bridge with ID %r" % id) + "from Bridge with ID %r" % ID) num_or_address_lines += 1 elif ID and timestamp and line.startswith("s "): flags = line.split() yield ID, ("Running" in flags), ("Stable" in flags), or_addresses, timestamp - # add or update BridgeHistory entries into the database - # XXX: what do we do with all these or_addresses? - # The bridge stability metrics are only concerned with a single ip:port - # So for now, we will only consider the bridges primary IP:port - bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) - timestamp = ID = None num_or_address_lines = 0 or_addresses = {}

1 0