commit ce065946118bda2901eeed03d7f842f9dfd415dd Author: Tom Ritter <tom@ritter.vg> Date: Fri Jul 1 17:49:06 2016 -0500 Track the number of known relays in addition to running relays. Store the data in the SQLite DB after retrieving the data --- .gitignore | 1 + parseOldConsensuses.py | 9 ++++++--- write_website.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index ad4f990..0fa8066 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ *.swp stem out/index.html +out/graphs.html out/consensus-health* out/download-stats.csv \ No newline at end of file diff --git a/parseOldConsensuses.py b/parseOldConsensuses.py index a616014..e45fd27 100755 --- a/parseOldConsensuses.py +++ b/parseOldConsensuses.py @@ -70,8 +70,8 @@ def main(dir): dirauth_columns = "" dirauth_columns_questions = "" for d in dirAuths: - dirauth_columns += d + "_running integer, " + d + "_bwauth integer, " - dirauth_columns_questions += ", ?, ?" + dirauth_columns += d + "_known integer, " + d + "_running integer, " + d + "_bwauth integer, " + dirauth_columns_questions += ",?,?,?" votes = {} for root, dirs, files in os.walk(dir): @@ -97,8 +97,9 @@ def main(dir): print "Found two votes for dirauth " + dirauth + " and time " + filepath votes[voteTime][dirauth]['present'] = 1 - votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l', shell=True)) + votes[voteTime][dirauth]['known'] = int(subprocess.check_output('egrep "^r " "' + filepath + '" | wc -l', shell=True)) votes[voteTime][dirauth]['running'] = int(subprocess.check_output('egrep "^s " "' + filepath + '" | grep " Running" | wc -l', shell=True)) + votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l', shell=True)) dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " + dirauth_columns + "PRIMARY KEY(date ASC))") dbc.commit() @@ -112,11 +113,13 @@ def main(dir): insertValues = [t] for d in dirAuths: if d in votes[t]: + insertValues.append(votes[t][d]['known']) insertValues.append(votes[t][d]['running']) insertValues.append(votes[t][d]['bwlines']) else: insertValues.append(None) insertValues.append(None) + insertValues.append(None) dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?" + dirauth_columns_questions + ")", insertValues) dbc.commit() diff --git a/write_website.py b/write_website.py index 85cca00..e47d9a3 100755 --- a/write_website.py +++ b/write_website.py @@ -9,6 +9,7 @@ Performs a variety of checks against the present votes and consensus. import os import sys import time +import sqlite3 import datetime import operator import traceback @@ -44,6 +45,8 @@ downloader = stem.descriptor.remote.DescriptorDownloader( def directory_authorities(): return dict((k, v) for (k, v) in DIRECTORY_AUTHORITIES.items() if k not in CONFIG['ignored_authorities']) +def unix_time(dt): + return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0 def main(): # loads configuration data @@ -59,6 +62,37 @@ def main(): f.write("%s,%i,%i\n" % (ds, time.time() * 1000, int(consensus_fetching_runtimes[ds] * 1000))) f.close() + dbc = sqlite3.connect(os.path.join('data', 'historical.db')) + + #Calculate the number of known and measured relays for each dirauth and insert it into the database + databaseDirAuths = "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ") + data = {} + for dirauth_nickname in votes: + vote = votes[dirauth_nickname] + + runningRelays = 0 + bandwidthWeights = 0 + for r in vote.routers.values(): + if r.measured >= 0L: + bandwidthWeights += 1 + if u'Running' in r.flags: + runningRelays += 1 + data[dirauth_nickname] = {'known' : len(vote.routers.values()), 'running' : runningRelays, 'bwlines' : bandwidthWeights} + + insertValues = [unix_time(consensuses.values()[0].valid_after)] + for dirauth_nickname in databaseDirAuths: + if dirauth_nickname in votes: + insertValues.append(data[dirauth_nickname]['known']) + insertValues.append(data[dirauth_nickname]['running']) + insertValues.append(data[dirauth_nickname]['bwlines']) + else: + insertValues.append(None) + insertValues.append(None) + insertValues.append(None) + + dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", insertValues) + dbc.commit() + # great for debugging #import pickle #pickle.dump(consensuses, open('consensus.p', 'wb'))