[tor-commits] [depictor/master] Refactor to use the config file more

[tom at torproject.org]

commit f5071fd7bdfe716c81c9761defb0baa53134e51d
Author: Tom <tom at ritter.vg>
Date:   Mon Oct 24 13:00:11 2016 -0500

    Refactor to use the config file more
    
    Remove hardcoded historical dirauths and put them into the config file.
    Remove hardcoded brideauths and put them there
    Add maatuska as a bwauth
---
 data/consensus.cfg | 20 ++++++++++++++++++++
 website.py         | 11 +++++++++--
 write_website.py   | 15 ++++++++++-----
 3 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/data/consensus.cfg b/data/consensus.cfg
index c57144a..8452beb 100644
--- a/data/consensus.cfg
+++ b/data/consensus.cfg
@@ -8,6 +8,26 @@ bandwidth_authorities moria1
 bandwidth_authorities gabelmoo
 bandwidth_authorities tor26
 bandwidth_authorities longclaw
+bandwidth_authorities maatuska
+
+# all current/historical bridge auths
+
+historical_bridge_authorities tonga
+historical_bridge_authorities bifrost
+
+# for creating tables in historical.db
+
+historical_dirauths faravahar
+historical_dirauths gabelmoo
+historical_dirauths dizum
+historical_dirauths moria1
+historical_dirauths urras
+historical_dirauths maatuska
+historical_dirauths longclaw
+historical_dirauths tor26
+historical_dirauths dannenberg
+historical_dirauths turtles
+
 
 # recognized tor consensus parameters
 
diff --git a/website.py b/website.py
index ba16c32..7b79ec9 100755
--- a/website.py
+++ b/website.py
@@ -18,7 +18,10 @@ class WebsiteWriter:
 	consensus = None
 	votes = None
 	fallback_dirs = None
+	config_set = False
 	known_authorities = []
+	historical_bridge_authorities = []
+	bandwidth_authorities = []
 	consensus_expirey = datetime.timedelta(hours=3)
 	directory_key_warning_time = datetime.timedelta(days=14)
 	known_params = []
@@ -46,11 +49,13 @@ class WebsiteWriter:
 		self.site.close()
 
 	def set_consensuses(self, c):
+		if not self.config_set:
+			raise Exception("Set config before calling")
 		self.consensuses = c
 		self.consensus = max(c.itervalues(), key=operator.attrgetter('valid_after'))
-		self.known_authorities = set([r.nickname for r in self.consensus.routers.values() if 'Authority' in r.flags and r.nickname != "Tonga" and r.nickname != "Bifroest"])
+		self.known_authorities = set([r.nickname for r in self.consensus.routers.values() if 'Authority' in r.flags and r.nickname not in self.historical_bridge_authorities])
 		self.known_authorities.update([r.nickname for r in self.consensus.directory_authorities])
-		self.known_authorities.update([r for r in stem.descriptor.remote.get_authorities().keys() if r != "Tonga" and r != "Bifroest"])
+		self.known_authorities.update([r for r in stem.descriptor.remote.get_authorities().keys() if r not in self.historical_bridge_authorities])
 	def set_votes(self, v):
 		self.votes = v
 	def set_consensus_expirey(self, timedelta):
@@ -58,7 +63,9 @@ class WebsiteWriter:
 	def set_directory_key_warning_time(self, timedelta):
 		self.directory_key_warning_time = timedelta
 	def set_config(self, config):
+		self.config_set = True
 		self.known_params = config['known_params']
+		self.historical_bridge_authorities = config['historical_bridge_authorities']
 		self.bandwidth_authorities = config['bandwidth_authorities']
 	def set_fallback_dirs(self, fallback_dirs):
 		self.fallback_dirs = fallback_dirs
diff --git a/write_website.py b/write_website.py
index d76d242..70edf3e 100755
--- a/write_website.py
+++ b/write_website.py
@@ -33,6 +33,8 @@ CONFIG = stem.util.conf.config_dict('consensus', {
 	'ignored_authorities': [],
 	'bandwidth_authorities': [],
 	'known_params': [],
+	'historical_dirauths' : [],
+	'historical_bridge_authorities' : []
 })
 
 downloader = stem.descriptor.remote.DescriptorDownloader(
@@ -107,7 +109,6 @@ def main():
 	f.close()
 
 	# Calculate the number of known and measured relays for each dirauth and insert it into the database
-	databaseDirAuths = "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ")
 	data = {}
 	for dirauth_nickname in votes:
 		vote = votes[dirauth_nickname]
@@ -122,7 +123,9 @@ def main():
 		data[dirauth_nickname] = {'known' : len(vote.routers.values()), 'running' : runningRelays, 'bwlines' : bandwidthWeights}
 
 	insertValues = [unix_time(consensuses.values()[0].valid_after)]
-	for dirauth_nickname in databaseDirAuths:
+	createColumns = ""
+	for dirauth_nickname in CONFIG['historical_dirauths']:
+		createColumns += dirauth_nickname + "_known integer, " + dirauth_nickname + "_running integer, " + dirauth_nickname + "_bwauth integer, "
 		if dirauth_nickname in votes:
 			insertValues.append(data[dirauth_nickname]['known'])
 			insertValues.append(data[dirauth_nickname]['running'])
@@ -132,7 +135,9 @@ def main():
 			insertValues.append(None)
 			insertValues.append(None)
 
-	dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
+	dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " + createColumns + " PRIMARY KEY(date ASC));")
+	dbc.commit()
+
 	dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", insertValues)
 	dbc.commit()
 
@@ -157,10 +162,10 @@ def main():
 
 	# produces the website
 	w = WebsiteWriter()
+	w.set_config(CONFIG)
 	w.set_consensuses(consensuses)
 	w.set_votes(votes)
 	w.set_fallback_dirs(fallback_dirs)
-	w.set_config(CONFIG)
 	w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'consensus-health.html'), True)
 	w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'index.html'), False)
 
@@ -169,10 +174,10 @@ def main():
 
 	# produces the website
 	g = GraphWriter()
+	g.set_config(CONFIG)
 	g.set_consensuses(consensuses)
 	g.set_votes(votes)
 	g.set_fallback_dirs(fallback_dirs)
-	g.set_config(CONFIG)
 	g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html'))
 
 	del g