commit 59bcda62f86ffbcc2b2c75e42a3a12d1150e1dae Author: aagbsn <aagbsn@extc.org> Date: Sun Nov 4 07:47:02 2012 -0800 5482 - Parse timestamp from networkstatus Bridge descriptors are only updated every 12-18 hours, but ns is updated every 30 minutes. --- lib/bridgedb/Bridges.py | 31 +++++++++++++++++-------------- lib/bridgedb/Main.py | 11 ++++++++--- lib/bridgedb/Tests.py | 8 ++++---- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 223a78d..822385f 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -329,7 +329,7 @@ def parseDescFile(f, bridge_purpose='bridge'): PORT = a number between 1 and 65535 inclusive. """ - nickname = ip = orport = fingerprint = purpose = timestamp = None + nickname = ip = orport = fingerprint = purpose = None num_or_address_lines = 0 or_addresses = {} @@ -348,19 +348,13 @@ def parseDescFile(f, bridge_purpose='bridge'): orport = int(items[3]) elif line.startswith("fingerprint "): fingerprint = line[12:].replace(" ", "") - elif line.startswith("published "): - if line.startswith("published "): - try: - timestamp = time.strptime(line[10:],"%Y-%m-%d %H:%M:%S") - timestamp = time.mktime(timestamp) - except ValueError: timestamp = None elif line.startswith("router-signature"): purposeMatches = (purpose == bridge_purpose or bridge_purpose is None) - if purposeMatches and nickname and ip and orport and fingerprint and timestamp: + if purposeMatches and nickname and ip and orport and fingerprint: b = Bridge(nickname, ipaddr.IPAddress(ip), orport, fingerprint) b.assertOK() - yield b, timestamp - nickname = ip = orport = fingerprint = purpose = timestamp = None + yield b + nickname = ip = orport = fingerprint = purpose = None class PortList: """ container class for port ranges @@ -540,7 +534,7 @@ def parseExtraInfoFile(f): def parseStatusFile(f): """DOCDOC""" - ID = None + timestamp = ID = None num_or_address_lines = 0 or_addresses = {} for line in f: @@ -551,8 +545,11 @@ def parseStatusFile(f): if line.startswith("r "): try: ID = binascii.a2b_base64(line.split()[2]+"=") + timestamp = time.strptime(line.split()[4],"%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timestamp) except binascii.Error: logging.warn("Unparseable base64 ID %r", line.split()[2]) + except ValueError: timestamp = None elif ID and line.startswith("a "): if num_or_address_lines < 8: @@ -567,10 +564,16 @@ def parseStatusFile(f): "from Bridge with ID %r" % id) num_or_address_lines += 1 - elif ID and line.startswith("s "): + elif ID and timestamp and line.startswith("s "): flags = line.split() - yield ID, ("Running" in flags), ("Stable" in flags), or_addresses - ID = None + yield ID, ("Running" in flags), ("Stable" in flags), or_addresses, timestamp + # add or update BridgeHistory entries into the database + # XXX: what do we do with all these or_addresses? + # The bridge stability metrics are only concerned with a single ip:port + # So for now, we will only consider the bridges primary IP:port + bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) + + timestamp = ID = None num_or_address_lines = 0 or_addresses = {} diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 56a52dd..808da43 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -168,16 +168,18 @@ def load(cfg, splitter, clear=False): addresses = {} if hasattr(cfg, "STATUS_FILE"): f = open(cfg.STATUS_FILE, 'r') - for ID, running, stable, or_addresses in Bridges.parseStatusFile(f): + for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(f): status[ID] = running, stable addresses[ID] = or_addresses + if ID in timestamps.keys(): timestamps[ID].append(timestamp) + else: timestamps[ID] = [timestamp] #transports[ID] = transports f.close() bridges = {} db = bridgedb.Storage.getDB() for fname in cfg.BRIDGE_FILES: f = open(fname, 'r') - for bridge, timestamp in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): + for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): bridges[bridge.getID()] = bridge s = status.get(bridge.getID()) if s is not None: @@ -189,7 +191,10 @@ def load(cfg, splitter, clear=False): # XXX: what do we do with all these or_addresses? # The bridge stability metrics are only concerned with a single ip:port # So for now, we will only consider the bridges primary IP:port - bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) + ts = timestamps[ID][:] + ts.sort() + for timestamp in ts: + bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) f.close() # read pluggable transports from extra-info document # XXX: should read from networkstatus after bridge-authority diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 505a7b7..79638f4 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -554,7 +554,7 @@ class ParseDescFileTests(unittest.TestCase): test+=gettimestamp() test+="router-signature\n" - bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: @@ -569,7 +569,7 @@ class ParseDescFileTests(unittest.TestCase): test+=gettimestamp() test+= "router-signature\n" - bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: @@ -584,7 +584,7 @@ class ParseDescFileTests(unittest.TestCase): test+=gettimestamp() test+= "router-signature\n" - bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: @@ -599,7 +599,7 @@ class ParseDescFileTests(unittest.TestCase): test+=gettimestamp() test+= "router-signature\n" - bs = [b for b,_ in bridgedb.Bridges.parseDescFile(test.split('\n'))] + bs = [b for b in bridgedb.Bridges.parseDescFile(test.split('\n'))] self.assertEquals(len(bs), 100) for b in bs: