[tor-commits] [depictor/master] Track the number of known relays in addition to running relays. Store the data in the SQLite DB after retrieving the data

[tom at torproject.org]

commit ce065946118bda2901eeed03d7f842f9dfd415dd
Author: Tom Ritter <tom at ritter.vg>
Date:   Fri Jul 1 17:49:06 2016 -0500

    Track the number of known relays in addition to running relays. Store the data in the SQLite DB after retrieving the data
---
 .gitignore             |  1 +
 parseOldConsensuses.py |  9 ++++++---
 write_website.py       | 34 ++++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index ad4f990..0fa8066 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,5 +3,6 @@
 *.swp
 stem
 out/index.html
+out/graphs.html
 out/consensus-health*
 out/download-stats.csv
\ No newline at end of file
diff --git a/parseOldConsensuses.py b/parseOldConsensuses.py
index a616014..e45fd27 100755
--- a/parseOldConsensuses.py
+++ b/parseOldConsensuses.py
@@ -70,8 +70,8 @@ def main(dir):
 	dirauth_columns = ""
 	dirauth_columns_questions = ""
 	for d in dirAuths:
-		dirauth_columns += d + "_running integer, " + d + "_bwauth integer, "
-		dirauth_columns_questions += ", ?, ?"
+		dirauth_columns += d + "_known integer, " + d + "_running integer, " + d + "_bwauth integer, "
+		dirauth_columns_questions += ",?,?,?"
 
 	votes = {}
 	for root, dirs, files in os.walk(dir):
@@ -97,8 +97,9 @@ def main(dir):
 				print "Found two votes for dirauth " + dirauth + " and time " + filepath
 
 			votes[voteTime][dirauth]['present'] = 1
-			votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l', shell=True))
+			votes[voteTime][dirauth]['known'] = int(subprocess.check_output('egrep "^r " "' + filepath + '" | wc -l', shell=True))
 			votes[voteTime][dirauth]['running'] = int(subprocess.check_output('egrep "^s " "' + filepath + '" | grep " Running" | wc -l', shell=True))
+			votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l', shell=True))
 
 	dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " + dirauth_columns + "PRIMARY KEY(date ASC))")
 	dbc.commit()
@@ -112,11 +113,13 @@ def main(dir):
 		insertValues = [t]
 		for d in dirAuths:
 			if d in votes[t]:
+				insertValues.append(votes[t][d]['known'])
 				insertValues.append(votes[t][d]['running'])
 				insertValues.append(votes[t][d]['bwlines'])
 			else:
 				insertValues.append(None)
 				insertValues.append(None)
+				insertValues.append(None)
 
 		dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?" + dirauth_columns_questions + ")", insertValues)
 		dbc.commit()
diff --git a/write_website.py b/write_website.py
index 85cca00..e47d9a3 100755
--- a/write_website.py
+++ b/write_website.py
@@ -9,6 +9,7 @@ Performs a variety of checks against the present votes and consensus.
 import os
 import sys
 import time
+import sqlite3
 import datetime
 import operator
 import traceback
@@ -44,6 +45,8 @@ downloader = stem.descriptor.remote.DescriptorDownloader(
 def directory_authorities():
 	return dict((k, v) for (k, v) in DIRECTORY_AUTHORITIES.items() if k not in CONFIG['ignored_authorities'])
 
+def unix_time(dt):
+    return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0
 
 def main():
 	# loads configuration data
@@ -59,6 +62,37 @@ def main():
 		f.write("%s,%i,%i\n" % (ds, time.time() * 1000, int(consensus_fetching_runtimes[ds] * 1000)))
 	f.close()
 
+	dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
+
+	#Calculate the number of known and measured relays for each dirauth and insert it into the database
+	databaseDirAuths = "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ")
+	data = {}
+	for dirauth_nickname in votes:
+		vote = votes[dirauth_nickname]
+				
+		runningRelays    = 0
+		bandwidthWeights = 0
+		for r in vote.routers.values():
+			if r.measured >= 0L:
+				bandwidthWeights += 1
+			if u'Running' in r.flags:
+				runningRelays += 1
+		data[dirauth_nickname] = {'known' : len(vote.routers.values()), 'running' : runningRelays, 'bwlines' : bandwidthWeights}
+
+	insertValues = [unix_time(consensuses.values()[0].valid_after)]
+	for dirauth_nickname in databaseDirAuths:
+		if dirauth_nickname in votes:
+			insertValues.append(data[dirauth_nickname]['known'])
+			insertValues.append(data[dirauth_nickname]['running'])
+			insertValues.append(data[dirauth_nickname]['bwlines'])
+		else:
+			insertValues.append(None)
+			insertValues.append(None)
+			insertValues.append(None)
+
+	dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", insertValues)
+	dbc.commit()
+
 	# great for debugging
 	#import pickle
 	#pickle.dump(consensuses, open('consensus.p', 'wb'))