commit 97bea9d85284ef38b589cd75d599d39ef5cd64db
Author: Tom Ritter <tom(a)ritter.vg>
Date: Tue Dec 21 21:25:46 2021 -0500
Update the parseOldconsensuses script
Make print statements python3-compatible
DescribtorReader was removed from stem, replace it
Add bastet to the known dirauths
convert tabs to spaces and completely fuck up the diff
---
parseOldConsensuses.py | 528 +++++++++++++++++++++++++------------------------
1 file changed, 265 insertions(+), 263 deletions(-)
diff --git a/parseOldConsensuses.py b/parseOldConsensuses.py
index defa2f5..e72d7a9 100755
--- a/parseOldConsensuses.py
+++ b/parseOldConsensuses.py
@@ -18,293 +18,295 @@ import stem.util.conf
import stem.util.enum
from stem import Flag
-from stem.descriptor.reader import DescriptorReader
-from stem.util.lru_cache import lru_cache
def get_dirauths_in_tables():
- return "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ")
+ return "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles, bastet".split(", ")
def get_dirauth_from_filename(filename):
- key = filename.split('-')
- if len(key) < 9:
- raise Exception("Strange filename: " + filename)
-
- key = key[-2]
- if key == "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97":
- return "faravahar"
- elif key == "ED03BB616EB2F60BEC80151114BB25CEF515B226":
- return "gabelmoo"
- elif key == "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58":
- return "dizum"
- elif key == "D586D18309DED4CD6D57C18FDB97EFA96D330566":
- return "moria1"
- elif key == "80550987E1D626E3EBA5E5E75A458DE0626D088C":
- return "urras"
- elif key == "49015F787433103580E3B66A1707A00E60F2D15B":
- return "maatuska"
- elif key == "23D15D965BC35114467363C165C4F724B64B4F66":
- return "longclaw"
- elif key == "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4":
- return "tor26"
- elif key == "0232AF901C31A04EE9848595AF9BB7620D4C5B2E" or key == "585769C78764D58426B8B52B6651A5A71137189A":
- return "dannenberg"
- elif key == "27B6B5996C426270A5C95488AA5BCEB6BCC86956":
- return "turtles"
- else:
- raise Exception("Unexpcected dirauth key: " + key + " " + filename)
+ key = filename.split('-')
+ if len(key) < 9:
+ raise Exception("Strange filename: " + filename)
+
+ key = key[-2]
+ if key == "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97":
+ return "faravahar"
+ elif key == "ED03BB616EB2F60BEC80151114BB25CEF515B226":
+ return "gabelmoo"
+ elif key == "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58":
+ return "dizum"
+ elif key == "D586D18309DED4CD6D57C18FDB97EFA96D330566":
+ return "moria1"
+ elif key == "80550987E1D626E3EBA5E5E75A458DE0626D088C":
+ return "urras"
+ elif key == "49015F787433103580E3B66A1707A00E60F2D15B":
+ return "maatuska"
+ elif key == "23D15D965BC35114467363C165C4F724B64B4F66":
+ return "longclaw"
+ elif key == "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4":
+ return "tor26"
+ elif key == "0232AF901C31A04EE9848595AF9BB7620D4C5B2E" or key == "585769C78764D58426B8B52B6651A5A71137189A":
+ return "dannenberg"
+ elif key == "27B6B5996C426270A5C95488AA5BCEB6BCC86956":
+ return "turtles"
+ elif key == "27102BC123E7AF1D4741AE047E160C91ADC76B21":
+ return "bastet"
+ else:
+ raise Exception("Unexpcected dirauth key: " + key + " " + filename)
def unix_time(dt):
return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0
def ut_to_datetime(ut):
- return datetime.datetime.utcfromtimestamp(ut / 1000)
+ return datetime.datetime.utcfromtimestamp(ut / 1000)
def ut_to_datetime_format(ut):
- return ut_to_datetime(ut).strftime("%Y-%m-%d-%H-%M-%S")
+ return ut_to_datetime(ut).strftime("%Y-%m-%d-%H-%M-%S")
def get_time_from_filename(filename):
- voteTime = filename.split('-')
- if len(voteTime) < 7:
- raise Exception("Strange filename: " + filename)
+ voteTime = filename.split('-')
+ if len(voteTime) < 7:
+ raise Exception("Strange filename: " + filename)
- v = [int(x) for x in filename.split('-')[0:6]]
- voteTime = datetime.datetime(v[0], v[1], v[2], v[3], v[4], v[5])
- voteTime = unix_time(voteTime)
- return voteTime
+ v = [int(x) for x in filename.split('-')[0:6]]
+ voteTime = datetime.datetime(v[0], v[1], v[2], v[3], v[4], v[5])
+ voteTime = unix_time(voteTime)
+ return voteTime
def dirauth_relay_votes(directory, dirAuths, dbc):
- dirauth_columns = ""
- dirauth_columns_questions = ""
- for d in dirAuths:
- dirauth_columns += d + "_known integer, " + d + "_running integer, " + d + "_bwauth integer, "
- dirauth_columns_questions += ",?,?,?"
-
- dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " + dirauth_columns + "PRIMARY KEY(date ASC))")
- dbc.commit()
-
- votes = {}
- for root, dirs, files in os.walk(directory):
- for f in files:
- filepath = os.path.join(root, f)
- print filepath
-
- if '"' in f:
- raise Exception("Potentially malicious filename")
- elif "votes-" in f and ".tar" in f:
- continue
- elif "consensuses-" in f and ".tar" in f:
- continue
- elif "-vote-" not in f:
- continue
-
- voteTime = get_time_from_filename(f)
- if voteTime not in votes:
- votes[voteTime] = {}
-
- dirauth = get_dirauth_from_filename(f)
- if dirauth not in dirAuths:
- raise Exception("Found a dirauth I don't know about (probably spelling): " + dirauth)
- elif dirauth not in votes[voteTime]:
- votes[voteTime][dirauth] = {}
- else:
- print "Found two votes for dirauth " + dirauth + " and time " + filepath
-
- votes[voteTime][dirauth]['present'] = 1
- votes[voteTime][dirauth]['known'] = int(subprocess.check_output('egrep "^r " "' + filepath + '" | wc -l', shell=True))
- votes[voteTime][dirauth]['running'] = int(subprocess.check_output('egrep "^s " "' + filepath + '" | grep " Running" | wc -l', shell=True))
- votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l', shell=True))
-
- for t in votes:
- print ut_to_datetime(t)
- print "\t", len(votes[t])
- for d in votes[t]:
- print "\t", d, votes[t][d]['bwlines'], votes[t][d]['running']
-
- insertValues = [t]
- for d in dirAuths:
- if d in votes[t]:
- insertValues.append(votes[t][d]['known'])
- insertValues.append(votes[t][d]['running'])
- insertValues.append(votes[t][d]['bwlines'])
- else:
- insertValues.append(None)
- insertValues.append(None)
- insertValues.append(None)
-
- dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?" + dirauth_columns_questions + ")", insertValues)
- dbc.commit()
+ dirauth_columns = ""
+ dirauth_columns_questions = ""
+ for d in dirAuths:
+ dirauth_columns += d + "_known integer, " + d + "_running integer, " + d + "_bwauth integer, "
+ dirauth_columns_questions += ",?,?,?"
+
+ dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " + dirauth_columns + "PRIMARY KEY(date ASC))")
+ dbc.commit()
+
+ votes = {}
+ for root, dirs, files in os.walk(directory):
+ for f in files:
+ filepath = os.path.join(root, f)
+ print(filepath)
+
+ if '"' in f:
+ raise Exception("Potentially malicious filename")
+ elif "votes-" in f and ".tar" in f:
+ continue
+ elif "consensuses-" in f and ".tar" in f:
+ continue
+ elif "-vote-" not in f:
+ continue
+
+ voteTime = get_time_from_filename(f)
+ if voteTime not in votes:
+ votes[voteTime] = {}
+
+ dirauth = get_dirauth_from_filename(f)
+ if dirauth not in dirAuths:
+ raise Exception("Found a dirauth I don't know about (probably spelling): " + dirauth)
+ elif dirauth not in votes[voteTime]:
+ votes[voteTime][dirauth] = {}
+ else:
+ print("Found two votes for dirauth " + dirauth + " and time " + filepath)
+
+ votes[voteTime][dirauth]['present'] = 1
+ votes[voteTime][dirauth]['known'] = int(subprocess.check_output('egrep "^r " "' + filepath + '" | wc -l', shell=True))
+ votes[voteTime][dirauth]['running'] = int(subprocess.check_output('egrep "^s " "' + filepath + '" | grep " Running" | wc -l', shell=True))
+ votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l', shell=True))
+
+ for t in votes:
+ print(ut_to_datetime(t))
+ print("\t", len(votes[t]))
+ for d in votes[t]:
+ print("\t", d, votes[t][d]['bwlines'], votes[t][d]['running'])
+
+ insertValues = [t]
+ for d in dirAuths:
+ if d in votes[t]:
+ insertValues.append(votes[t][d]['known'])
+ insertValues.append(votes[t][d]['running'])
+ insertValues.append(votes[t][d]['bwlines'])
+ else:
+ insertValues.append(None)
+ insertValues.append(None)
+ insertValues.append(None)
+
+ dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?" + dirauth_columns_questions + ")", insertValues)
+ dbc.commit()
def bwauth_measurements(directory, dirAuths, dbc):
- #Find all the consensuses and votesrm
- votes = {}
- consensuses = {}
- for root, dirs, files in os.walk(directory):
- for f in files:
- filepath = os.path.join(root, f)
-
- if '"' in f:
- raise Exception("Potentially malicious filename")
- elif "votes-" in f and ".tar" in f:
- continue
- elif "consensuses-" in f and ".tar" in f:
- continue
-
- if "-consensus" in f:
- consensusTime = get_time_from_filename(f)
- if consensusTime not in consensuses:
- consensuses[consensusTime] = filepath
- else:
- print "Found two consensuses with the same time:", ut_to_datetime(consensusTime)
-
- #print "Consensus:", filepath
- elif "-vote-" in f:
- voteTime = get_time_from_filename(f)
-
- # Test to see if we already processed this one
- cur = dbc.cursor()
- cur.execute("SELECT * FROM bwauth_data WHERE date = ?", (voteTime,))
- if cur.fetchone():
- print "Skipping", f, "because we already processed it"
- continue
- elif voteTime not in votes:
- votes[voteTime] = {}
-
- dirauth = get_dirauth_from_filename(f)
-
- if dirauth not in dirAuths:
- raise Exception("Found a dirauth I don't know about (probably spelling): " + dirauth)
- elif dirauth not in votes[voteTime]:
- votes[voteTime][dirauth] = filepath
- else:
- print "Found two votes for dirauth " + dirauth + ":", filepath, "and", votes[voteTime][dirauth]
-
- #print "Vote:", dirauth, filepath
-
- #Make sure we have a consensus for each vote
- to_del = []
- for v in votes:
- if v not in consensuses:
- print "Have votes for time", ut_to_datetime(v), "but no consensus!"
- to_del.append(v)
- #sys.exit(1)
- for i in to_del:
- del votes[i]
-
- #Make the table
- bwauth_columns = ""
- bwauth_columns_questions = ""
- for d in dirAuths:
- bwauth_columns += d + "_above integer, " + d + "_shared integer, " + d + "_exclusive integer, " + d + "_below integer, " + d + "_unmeasured integer, "
- bwauth_columns_questions += ",?,?,?,?,?"
-
- dbc.execute("CREATE TABLE IF NOT EXISTS bwauth_data(date integer, " + bwauth_columns + "PRIMARY KEY(date ASC))")
- dbc.commit()
-
- reviewed = 0
- for v in votes:
- reviewed += 1
- print "Reviewing", consensuses[v], "(" + str(reviewed) + "/" + str(len(votes)) + ")"
-
- #Get the consensus data
- consensusRouters = {}
- with DescriptorReader(consensuses[v]) as reader:
- reader.register_skip_listener(my_listener)
- for relay in reader:
- consensusRouters[relay.fingerprint] = "Unmeasured" if relay.is_unmeasured else relay.bandwidth
-
- #The vote data
- bwauthVotes = {}
- for d in votes[v]:
- if d not in bwauthVotes:
- bwauthVotes[d] = {}
-
- measured_something = False
- with DescriptorReader(votes[v][d]) as reader:
- reader.register_skip_listener(my_listener)
- for relay in reader:
- if relay.measured:
- bwauthVotes[d][relay.fingerprint] = relay.measured
- measured_something = True
- if not measured_something:
- del bwauthVotes[d]
-
- #Now match them up and store the data
- thisConsensusResults = {}
- for r in consensusRouters:
- for d in bwauthVotes:
- had_any_value = False
- if d not in thisConsensusResults:
- thisConsensusResults[d] = {'unmeasured' : 0, 'above' : 0, 'below' : 0, 'exclusive' : 0 , 'shared' : 0}
-
- if consensusRouters[r] == "Unmeasured":
- continue
- elif r not in bwauthVotes[d]:
- had_any_value = True
- thisConsensusResults[d]['unmeasured'] += 1
- elif consensusRouters[r] < bwauthVotes[d][r]:
- had_any_value = True
- thisConsensusResults[d]['above'] += 1
- elif consensusRouters[r] > bwauthVotes[d][r]:
- had_any_value = True
- thisConsensusResults[d]['below'] += 1
- elif consensusRouters[r] == bwauthVotes[d][r] and \
- 1 == len([1 for d_i in bwauthVotes if d_i in bwauthVotes and r in bwauthVotes[d_i] and bwauthVotes[d_i][r] == consensusRouters[r]]):
- had_any_value = True
- thisConsensusResults[d]['exclusive'] += 1
- elif consensusRouters[r] == bwauthVotes[d][r] and \
- 1 != len([1 for d_i in bwauthVotes if d_i in bwauthVotes and r in bwauthVotes[d_i] and bwauthVotes[d_i][r] == consensusRouters[r] ]):
- had_any_value = True
- thisConsensusResults[d]['shared'] += 1
- else:
- print "What case am I in???"
- sys.exit(1)
-
- if not had_any_value:
- del thisConsensusResults[d]
-
- insertValues = [v]
- for d in dirAuths:
- if d in thisConsensusResults:
- insertValues.append(thisConsensusResults[d]['above'])
- insertValues.append(thisConsensusResults[d]['shared'])
- insertValues.append(thisConsensusResults[d]['exclusive'])
- insertValues.append(thisConsensusResults[d]['below'])
- insertValues.append(thisConsensusResults[d]['unmeasured'])
- else:
- insertValues.append(None)
- insertValues.append(None)
- insertValues.append(None)
- insertValues.append(None)
- insertValues.append(None)
-
- dbc.execute("INSERT OR REPLACE INTO bwauth_data VALUES (?" + bwauth_columns_questions + ")", insertValues)
- dbc.commit()
-
+ #Find all the consensuses and votesrm
+ votes = {}
+ consensuses = {}
+ for root, dirs, files in os.walk(directory):
+ for f in files:
+ filepath = os.path.join(root, f)
+
+ if '"' in f:
+ raise Exception("Potentially malicious filename")
+ elif "votes-" in f and ".tar" in f:
+ continue
+ elif "consensuses-" in f and ".tar" in f:
+ continue
+
+ if "-consensus" in f:
+ consensusTime = get_time_from_filename(f)
+ if consensusTime not in consensuses:
+ consensuses[consensusTime] = filepath
+ else:
+ print("Found two consensuses with the same time:", ut_to_datetime(consensusTime))
+
+ #print "Consensus:", filepath
+ elif "-vote-" in f:
+ voteTime = get_time_from_filename(f)
+
+ # Test to see if we already processed this one
+ cur = dbc.cursor()
+ cur.execute("SELECT * FROM bwauth_data WHERE date = ? AND faravahar_above IS NOT NULL", (voteTime,))
+ if cur.fetchone():
+ #print("Skipping", f, "because we already processed it")
+ continue
+ elif voteTime not in votes:
+ votes[voteTime] = {}
+
+ dirauth = get_dirauth_from_filename(f)
+
+ if dirauth not in dirAuths:
+ raise Exception("Found a dirauth I don't know about (probably spelling): " + dirauth)
+ elif dirauth not in votes[voteTime]:
+ votes[voteTime][dirauth] = filepath
+ else:
+ print("Found two votes for dirauth " + dirauth + ":", filepath, "and", votes[voteTime][dirauth])
+
+ #print "Vote:", dirauth, filepath
+
+ print("Found %s consensuses" % len(consensuses))
+ print("Found %s votes" % len(votes))
+
+ #Make sure we have a consensus for each vote
+ to_del = []
+ for v in votes:
+ if v not in consensuses:
+ print("Have votes for time", ut_to_datetime(v), "but no consensus!")
+ to_del.append(v)
+ #sys.exit(1)
+ for i in to_del:
+ del votes[i]
+
+ #Make the table
+ bwauth_columns = ""
+ bwauth_columns_questions = ""
+ for d in dirAuths:
+ bwauth_columns += d + "_above integer, " + d + "_shared integer, " + d + "_exclusive integer, " + d + "_below integer, " + d + "_unmeasured integer, "
+ bwauth_columns_questions += ",?,?,?,?,?"
+
+ dbc.execute("CREATE TABLE IF NOT EXISTS bwauth_data(date integer, " + bwauth_columns + "PRIMARY KEY(date ASC))")
+ dbc.commit()
+
+ reviewed = 0
+ for v in votes:
+ reviewed += 1
+ print("Reviewing", consensuses[v], "(" + str(reviewed) + "/" + str(len(votes)) + ")")
+
+ #Get the consensus data
+ consensusRouters = {}
+ reader = stem.descriptor.parse_file(consensuses[v])
+ for relay in reader:
+ consensusRouters[relay.fingerprint] = "Unmeasured" if relay.is_unmeasured else relay.bandwidth
+
+ #The vote data
+ bwauthVotes = {}
+ for d in votes[v]:
+ if d not in bwauthVotes:
+ bwauthVotes[d] = {}
+
+ measured_something = False
+ reader = stem.descriptor.parse_file(votes[v][d])
+ for relay in reader:
+ if relay.measured:
+ bwauthVotes[d][relay.fingerprint] = relay.measured
+ measured_something = True
+
+ if not measured_something:
+ del bwauthVotes[d]
+
+ #Now match them up and store the data
+ thisConsensusResults = {}
+ for r in consensusRouters:
+ for d in bwauthVotes:
+ had_any_value = False
+ if d not in thisConsensusResults:
+ thisConsensusResults[d] = {'unmeasured' : 0, 'above' : 0, 'below' : 0, 'exclusive' : 0 , 'shared' : 0}
+
+ if consensusRouters[r] == "Unmeasured":
+ continue
+ elif r not in bwauthVotes[d]:
+ had_any_value = True
+ thisConsensusResults[d]['unmeasured'] += 1
+ elif consensusRouters[r] < bwauthVotes[d][r]:
+ had_any_value = True
+ thisConsensusResults[d]['above'] += 1
+ elif consensusRouters[r] > bwauthVotes[d][r]:
+ had_any_value = True
+ thisConsensusResults[d]['below'] += 1
+ elif consensusRouters[r] == bwauthVotes[d][r] and \
+ 1 == len([1 for d_i in bwauthVotes if d_i in bwauthVotes and r in bwauthVotes[d_i] and bwauthVotes[d_i][r] == consensusRouters[r]]):
+ had_any_value = True
+ thisConsensusResults[d]['exclusive'] += 1
+ elif consensusRouters[r] == bwauthVotes[d][r] and \
+ 1 != len([1 for d_i in bwauthVotes if d_i in bwauthVotes and r in bwauthVotes[d_i] and bwauthVotes[d_i][r] == consensusRouters[r] ]):
+ had_any_value = True
+ thisConsensusResults[d]['shared'] += 1
+ else:
+ print("What case am I in???")
+ sys.exit(1)
+
+ if not had_any_value:
+ del thisConsensusResults[d]
+
+ insertValues = [v]
+ for d in dirAuths:
+ if d in thisConsensusResults:
+ insertValues.append(thisConsensusResults[d]['above'])
+ insertValues.append(thisConsensusResults[d]['shared'])
+ insertValues.append(thisConsensusResults[d]['exclusive'])
+ insertValues.append(thisConsensusResults[d]['below'])
+ insertValues.append(thisConsensusResults[d]['unmeasured'])
+ else:
+ insertValues.append(None)
+ insertValues.append(None)
+ insertValues.append(None)
+ insertValues.append(None)
+ insertValues.append(None)
+
+ dbc.execute("INSERT OR REPLACE INTO bwauth_data VALUES (?" + bwauth_columns_questions + ")", insertValues)
+ dbc.commit()
+
def my_listener(path, exception):
- print "Skipped!"
- print path
- print exception
+ print("Skipped!")
+ print(path)
+ print(exception)
def main(itype, directory):
- dirAuths = get_dirauths_in_tables()
- dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
+ dirAuths = get_dirauths_in_tables()
+ dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
- if itype == "dirauth_relay_votes":
- dirauth_relay_votes(directory, dirAuths, dbc)
- elif itype == "bwauth_measurements":
- bwauth_measurements(directory, dirAuths, dbc)
- else:
- print "Unknown ingestion type"
+ if itype == "dirauth_relay_votes":
+ dirauth_relay_votes(directory, dirAuths, dbc)
+ elif itype == "bwauth_measurements":
+ bwauth_measurements(directory, dirAuths, dbc)
+ else:
+ print("Unknown ingestion type")
if __name__ == '__main__':
- try:
- if len(sys.argv) != 3:
- print "Usage: ", sys.argv[0], "ingestion-type vote-directory"
- else:
- main(sys.argv[1], sys.argv[2])
- except:
- msg = "%s failed with:\n\n%s" % (sys.argv[0], traceback.format_exc())
- print "Error: %s" % msg
+ try:
+ if len(sys.argv) != 3:
+ print("Usage: ", sys.argv[0], "ingestion-type vote-directory")
+ else:
+ main(sys.argv[1], sys.argv[2])
+ except:
+ msg = "%s failed with:\n\n%s" % (sys.argv[0], traceback.format_exc())
+ print("Error: %s" % msg)
