[tor-commits] [depictor/master] Big refactor
tom at torproject.org
tom at torproject.org
Wed Nov 2 19:56:30 UTC 2016
commit b30311728584e0682c3d02079932611d3d547e1a
Author: Tom <tom at ritter.vg>
Date: Thu Oct 27 14:34:41 2016 -0500
Big refactor
- Put in a stub for putting your own DirAuths in
- Fix 20480 by lowercasing dirauth names consistenty
- Fix 20464 by making all operations with vote_data smarter.
- Insert with named columns
- Examine table structure and add columns if a dirauth gets added
- select named columns for the csv
- Remove last hardcoded dirauth stuff from graphs and make the colors dynamic
- Add a config option to ignore fallback directories (important for other networks)
- Remove super-redundant dirauth searching in website.py
- Move some stuff to a utility file
---
.gitignore | 4 +
data/consensus.cfg | 17 +--
graphs.py | 380 +++++++++++++++++++++++++++--------------------------
utility.py | 86 ++++++++++++
website.py | 43 +++---
write_website.py | 191 ++++++++++-----------------
6 files changed, 373 insertions(+), 348 deletions(-)
diff --git a/.gitignore b/.gitignore
index 45104d6..037b253 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,9 +2,13 @@
*.p
*.swp
stem
+
out/index.html
out/graphs.html
out/vote-stats.csv
out/consensus-health*
out/download-stats.csv
+out/fallback-dir-stats.csv
+out/historical.db
+
data/historical.db
\ No newline at end of file
diff --git a/data/consensus.cfg b/data/consensus.cfg
index c4a2142..381684a 100644
--- a/data/consensus.cfg
+++ b/data/consensus.cfg
@@ -1,19 +1,6 @@
-# directory authorities for which to not provide data
+# control consensus-health output
-ignored_authorities turtles
-
-# directory authorities that run bandwidth scanners
-
-bandwidth_authorities moria1
-bandwidth_authorities gabelmoo
-bandwidth_authorities tor26
-bandwidth_authorities longclaw
-bandwidth_authorities maatuska
-
-# all current/historical bridge auths
-
-historical_bridge_authorities tonga
-historical_bridge_authorities bifrost
+ignore_fallback_authorities False
# recognized tor consensus parameters
diff --git a/graphs.py b/graphs.py
index 9293b1f..6738a52 100755
--- a/graphs.py
+++ b/graphs.py
@@ -14,7 +14,7 @@ import stem.descriptor.remote
from base64 import b64decode
from website import WebsiteWriter
-from parseOldConsensuses import get_dirauths_in_tables
+from utility import get_dirauths, get_bwauths
class GraphWriter(WebsiteWriter):
def write_website(self, filename):
@@ -65,13 +65,14 @@ class GraphWriter(WebsiteWriter):
+ " text-align: center;\n"
+ " display: none;\n"
+ " }\n"
- + " .faravahar {\n"
+
+ + " .auth1 {\n"
+ " fill: none;\n"
+ " stroke: steelblue;\n"
+ " background-color: steelblue;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .gabelmoo, .orange {\n"
+ + " .auth2, .orange {\n"
+ " fill: none;\n"
+ " stroke: orange;\n"
+ " background-color: orange;\n"
@@ -80,13 +81,13 @@ class GraphWriter(WebsiteWriter):
+ " .orange {\n"
+ " fill: orange;"
+ " }\n"
- + " .moria1 {\n"
+ + " .auth3 {\n"
+ " fill: none;\n"
+ " stroke: yellow;\n"
+ " background-color: yellow;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .maatuska, .green {\n"
+ + " .auth4, .green {\n"
+ " fill: none;\n"
+ " stroke: green;\n"
+ " background-color: green;\n"
@@ -95,7 +96,7 @@ class GraphWriter(WebsiteWriter):
+ " .green {\n"
+ " fill: green;"
+ " }\n"
- + " .longclaw, .red {\n"
+ + " .auth5, .red {\n"
+ " fill: none;\n"
+ " stroke: red;\n"
+ " background-color: red;\n"
@@ -104,36 +105,37 @@ class GraphWriter(WebsiteWriter):
+ " .red {\n"
+ " fill: red;"
+ " }\n"
- + " .tor26 {\n"
+ + " .auth6 {\n"
+ " fill: none;\n"
+ " stroke: purple;\n"
+ " background-color: purple;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .urras {\n"
+ + " .auth7 {\n"
+ " fill: none;\n"
+ " stroke: black;\n"
+ " background-color: black;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .turtles {\n"
+ + " .auth8 {\n"
+ " fill: none;\n"
+ " stroke: #0000FF;\n"
+ " background-color: #0000FF;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .dizum {\n"
+ + " .auth9 {\n"
+ " fill: none;\n"
+ " stroke: limegreen;\n"
+ " background-color: limegreen;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .dannenberg {\n"
+ + " .auth10 {\n"
+ " fill: none;\n"
+ " stroke: pink;\n"
+ " background-color: pink;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
+
+ " </style>\n"
+ " <div class=\"center\">\n"
+ " <div class=\"main-column\">\n"
@@ -159,6 +161,9 @@ class GraphWriter(WebsiteWriter):
"""
Write the graphs of the fallback directory mirrors
"""
+ if self.config['ignore_fallback_authorities']:
+ return
+
self.site.write("<br>\n\n\n"
+ " <!-- ================================================================= -->"
+ "<a name=\"fallbackdirgraphs\">\n"
@@ -167,8 +172,7 @@ class GraphWriter(WebsiteWriter):
+ "<br>\n"
+ "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
+ " <colgroup>\n"
- + " <col width=\"160\">\n"
- + " <col width=\"640\">\n"
+ + " <col width=\"800\">\n"
+ " </colgroup>\n"
+ " <tr class=\"graphplaceholder\">\n"
+ " <td>\n"
@@ -185,19 +189,17 @@ class GraphWriter(WebsiteWriter):
self.site.write("</table>\n")
#-----------------------------------------------------------------------------------------
- def _write_number_of_relays_voted_about_graphs_spot(self, divName):
+ def _write_number_of_relays_voted_about_graphs_spot(self, divName, dirAuths):
self.site.write(" <tr>\n"
+ " <td>\n"
- + " <div id=\"" + str(divName) + "\" class=\"graphbox\">\n"
- + " <span class=\"moria1\" style=\"margin-left:5px\"> </span> Moria\n"
- + " <span class=\"faravahar\" style=\"margin-left:5px\"> </span> Faravahar\n"
- + " <span class=\"gabelmoo\" style=\"margin-left:5px\"> </span> Gabelmoo\n"
- + " <span class=\"maatuska\" style=\"margin-left:5px\"> </span> Maatuska\n"
- + " <span class=\"longclaw\" style=\"margin-left:5px\"> </span> Longclaw\n"
- + " <span class=\"tor26\" style=\"margin-left:5px\"> </span> tor26\n"
- + " <span class=\"dizum\" style=\"margin-left:5px\"> </span> dizum\n"
- + " <span class=\"dannenberg\" style=\"margin-left:5px\"> </span> dannenberg\n"
- + " </div>\n"
+ + " <div id=\"" + str(divName) + "\" class=\"graphbox\">\n")
+
+ i = 0
+ for d in dirAuths:
+ i += 1
+ self.site.write(" <span class=\"auth" + str(i) + "\" style=\"margin-left:5px\"> </span> " + d + "\n")
+
+ self.site.write(" </div>\n"
+ " </td>\n"
+ " </tr>\n")
def _write_number_of_relays_voted_about_graphs(self):
@@ -212,8 +214,7 @@ class GraphWriter(WebsiteWriter):
+ "<br>\n"
+ "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
+ " <colgroup>\n"
- + " <col width=\"160\">\n"
- + " <col width=\"640\">\n"
+ + " <col width=\"800\">\n"
+ " </colgroup>\n"
+ " <tr class=\"graphplaceholder\">\n"
+ " <td>\n"
@@ -222,31 +223,32 @@ class GraphWriter(WebsiteWriter):
+ " </div>\n"
+ " </td>\n"
+ " </tr>\n")
- self._write_number_of_relays_voted_about_graphs_spot("voted_total_1")
- self._write_number_of_relays_voted_about_graphs_spot("voted_total_2")
- self._write_number_of_relays_voted_about_graphs_spot("voted_total_3")
- self._write_number_of_relays_voted_about_graphs_spot("voted_total_4")
- self._write_number_of_relays_voted_about_graphs_spot("voted_running_1")
- self._write_number_of_relays_voted_about_graphs_spot("voted_running_2")
- self._write_number_of_relays_voted_about_graphs_spot("voted_running_3")
- self._write_number_of_relays_voted_about_graphs_spot("voted_running_4")
- self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_1")
- self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_2")
- self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_3")
- self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_4")
+ self._write_number_of_relays_voted_about_graphs_spot("voted_total_1", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_total_2", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_total_3", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_total_4", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_running_1", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_running_2", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_running_3", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_running_4", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_1", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_2", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_3", get_dirauths())
+ self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_4", get_dirauths())
self.site.write("</table>\n")
#-----------------------------------------------------------------------------------------
- def _write_bandwidth_scanner_graphs_spot(self, divName):
+ def _write_bandwidth_scanner_graphs_spot(self, divName, bwAuths):
self.site.write(" <tr>\n"
+ " <td>\n"
- + " <div id=\"" + str(divName) + "\" class=\"graphbox\">\n"
- + " <span class=\"moria1\" style=\"margin-left:5px\"> </span> Moria\n"
- + " <span class=\"faravahar\" style=\"margin-left:5px\"> </span> Faravahar\n"
- + " <span class=\"gabelmoo\" style=\"margin-left:5px\"> </span> Gabelmoo\n"
- + " <span class=\"maatuska\" style=\"margin-left:5px\"> </span> Maatuska\n"
- + " <span class=\"longclaw\" style=\"margin-left:5px\"> </span> Longclaw\n"
- + " </div>\n"
+ + " <div id=\"" + str(divName) + "\" class=\"graphbox\">\n")
+
+ i = 0
+ for d in bwAuths:
+ i += 1
+ self.site.write(" <span class=\"auth" + str(i) + "\" style=\"margin-left:5px\"> </span> " + d + "\n")
+
+ self.site.write(" </div>\n"
+ " </td>\n"
+ " </tr>\n")
def _write_bandwidth_scanner_graphs(self):
@@ -261,8 +263,7 @@ class GraphWriter(WebsiteWriter):
+ "<br>\n"
+ "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
+ " <colgroup>\n"
- + " <col width=\"160\">\n"
- + " <col width=\"640\">\n"
+ + " <col width=\"800\">\n"
+ " </colgroup>\n"
+ " <tr class=\"graphplaceholder\">\n"
+ " <td>\n"
@@ -271,10 +272,10 @@ class GraphWriter(WebsiteWriter):
+ " </div>\n"
+ " </td>\n"
+ " </tr>\n")
- self._write_bandwidth_scanner_graphs_spot("bwauth_measured_1")
- self._write_bandwidth_scanner_graphs_spot("bwauth_measured_2")
- self._write_bandwidth_scanner_graphs_spot("bwauth_measured_3")
- self._write_bandwidth_scanner_graphs_spot("bwauth_measured_4")
+ self._write_bandwidth_scanner_graphs_spot("bwauth_measured_1", get_bwauths())
+ self._write_bandwidth_scanner_graphs_spot("bwauth_measured_2", get_bwauths())
+ self._write_bandwidth_scanner_graphs_spot("bwauth_measured_3", get_bwauths())
+ self._write_bandwidth_scanner_graphs_spot("bwauth_measured_4", get_bwauths())
#self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_1")
#self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_2")
#self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_3")
@@ -289,10 +290,9 @@ class GraphWriter(WebsiteWriter):
HEIGHT = 500,
MARGIN = {top: 40, right: 40, bottom: 40, left: 40};
- var bwauths = ["faravahar","gabelmoo","moria1","maatuska","longclaw"];
- var dirauths = """ + str(get_dirauths_in_tables()) + """;
- dirauths.splice(dirauths.indexOf('urras'), 1);
- dirauths.splice(dirauths.indexOf('turtles'), 1);
+ var bwauths = """ + str(get_bwauths().keys()) + """;
+ var dirauths = """ + str(get_dirauths().keys()) + """;
+ var ignore_fallback_dirs = """ + str(self.config['ignore_fallback_authorities']).lower() + """;
var _getBandwidthDataValue = function(d, dirauth) { return d[dirauth + "_bwauth"]; }
var _getRunningDataValue = function(d, dirauth) { return d[dirauth + "_running"]; }
@@ -360,7 +360,7 @@ class GraphWriter(WebsiteWriter):
];
relays_done = false;
- fallbackdirs_done = false;
+ fallbackdirs_done = ignore_fallback_dirs;
fetch("vote-stats.csv").then(function(response) {
return response.text();
}).then(function(text) {
@@ -381,11 +381,11 @@ class GraphWriter(WebsiteWriter):
graph = GRAPHS_TO_GENERATE[g];
if(graph.data_slice+1 > data.length) {
- data_subset = data.slice(1);
+ data_subset = data.slice(0);
console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items...");
}
else
- data_subset = data.slice(1, graph.data_slice+1);
+ data_subset = data.slice(0, graph.data_slice+1);
data_subset.reverse();
// Calculate the Graph Boundaries -----------------------------------------
@@ -436,11 +436,12 @@ class GraphWriter(WebsiteWriter):
.domain([avg-(5*stddev), avg+(5*stddev)])
.range([HEIGHT, 0]);
+ var i = 1;
var lines = []
for(auth in graph.authorities)
{
this_auth = graph.authorities[auth];
- lines.push({auth: this_auth, line: (function(dirAuthClosure) {
+ lines.push({authIndex: i, line: (function(dirAuthClosure) {
return d3.line()
.defined(function(d) {
return d && graph.data_func(d, dirAuthClosure) &&
@@ -449,6 +450,7 @@ class GraphWriter(WebsiteWriter):
.x(function(d) { return x(d.date); })
.y(function(d) { return y(graph.data_func(d, dirAuthClosure)); });
})(this_auth)});
+ i++;
}
var svg = d3.select("#" + graph.div).append("svg")
@@ -470,7 +472,7 @@ class GraphWriter(WebsiteWriter):
for(l in lines)
{
svg.append("path")
- .attr("class", lines[l].auth)
+ .attr("class", "auth" + lines[l].authIndex)
.attr("d", lines[l].line);
}
@@ -496,136 +498,139 @@ class GraphWriter(WebsiteWriter):
});
- fetch("fallback-dir-stats.csv").then(function(response) {
- return response.text();
- }).then(function(text) {
- return d3.csvParse(text, function(d) {
- for(i in d) {
- if(i == "date")
- d[i] = new Date(Number(d[i]));
+ if(!ignore_fallback_dirs) {
+
+ fetch("fallback-dir-stats.csv").then(function(response) {
+ return response.text();
+ }).then(function(text) {
+ return d3.csvParse(text, function(d) {
+ for(i in d) {
+ if(i == "date")
+ d[i] = new Date(Number(d[i]));
+ else
+ d[i] = Number(d[i]);
+ }
+ return d;
+ });
+ }).then(function(data) {
+ var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' };
+ /*Pie Graph
+ data_subset = data.slice(0);
+ data_subset = [
+ {'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']},
+ {'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']},
+ {'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']},
+ ];
+ var data_func = function(d) { return d.value; };
+ var arcs = d3.pie()
+ .sort(null)
+ .value(data_func)(data_subset);
+
+ var svg = d3.select('#fallbackdirs_pie')
+ .append('svg')
+ .attr('width', WIDTH)
+ .attr('height', HEIGHT)
+ .append('g')
+ .attr('transform', 'translate(' + (WIDTH / 2) + ',' + (HEIGHT / 2) + ')');
+
+ var arc = d3.arc()
+ .innerRadius(0)
+ .outerRadius(100);
+
+ var path = svg.selectAll('path')
+ .data(arcs)
+ .enter()
+ .append('path')
+ .attr('d', arc)
+ .attr('class', function(d, i) {
+ return key_to_color(d.data.label);
+ });*/
+
+ //Line Graphs
+ for(g in FALLBACK_GRAPHS_TO_GENERATE)
+ {
+ graph = FALLBACK_GRAPHS_TO_GENERATE[g];
+
+ if(graph.data_slice+1 > data.length) {
+ data_subset = data.slice(0);
+ console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items...");
+ }
else
- d[i] = Number(d[i]);
- }
- return d;
- });
- }).then(function(data) {
- var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' };
- /*Pie Graph
- data_subset = data.slice(0);
- data_subset = [
- {'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']},
- {'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']},
- {'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']},
- ];
- var data_func = function(d) { return d.value; };
- var arcs = d3.pie()
- .sort(null)
- .value(data_func)(data_subset);
-
- var svg = d3.select('#fallbackdirs_pie')
- .append('svg')
- .attr('width', WIDTH)
- .attr('height', HEIGHT)
- .append('g')
- .attr('transform', 'translate(' + (WIDTH / 2) + ',' + (HEIGHT / 2) + ')');
-
- var arc = d3.arc()
- .innerRadius(0)
- .outerRadius(100);
-
- var path = svg.selectAll('path')
- .data(arcs)
- .enter()
- .append('path')
- .attr('d', arc)
- .attr('class', function(d, i) {
- return key_to_color(d.data.label);
- });*/
-
- //Line Graphs
- for(g in FALLBACK_GRAPHS_TO_GENERATE)
- {
- graph = FALLBACK_GRAPHS_TO_GENERATE[g];
+ data_subset = data.slice(0, graph.data_slice);
+ data_subset.reverse();
+
+ max = 0
+ for(d in data_subset) {
+ x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing'];
+ if(x > max)
+ max = x;
+ }
- if(graph.data_slice+1 > data.length) {
- data_subset = data.slice(0);
- console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items...");
- }
- else
- data_subset = data.slice(0, graph.data_slice);
- data_subset.reverse();
-
- max = 0
- for(d in data_subset) {
- x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing'];
- if(x > max)
- max = x;
+ var x = d3.scaleTime()
+ .domain([data_subset[0].date, data_subset[data_subset.length-1].date])
+ .range([0, WIDTH]);
+
+ var y = d3.scaleLinear()
+ .domain([0, max])
+ .range([HEIGHT, 0]);
+
+ var stack = d3.stack()
+ .keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"])
+ .order(d3.stackOrderNone)
+ .offset(d3.stackOffsetNone);
+
+ var area = d3.area()
+ .x(function(d, i) { return x(d.data.date); })
+ .y0(function(d) { return y(d[0]); })
+ .y1(function(d) { return y(d[1]); });
+
+ var svg = d3.select("#" + graph.div).append("svg")
+ .attr("width", WIDTH + MARGIN.left + MARGIN.right)
+ .attr("height", HEIGHT + MARGIN.top + MARGIN.bottom)
+ .append("g")
+ .attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")");
+
+ var layer = svg.selectAll(".layer")
+ .data(stack(data_subset))
+ .enter().append("g")
+ //.attr("class", "layer");
+
+ layer.append("path")
+ //.attr("class", "area")
+ .attr("class", function(d) { return key_to_color(d.key); })
+ .attr("d", area);
+
+ svg.append("g")
+ .attr("class", "axis axis--x")
+ .attr("transform", "translate(0," + HEIGHT + ")")
+ .call(d3.axisBottom().scale(x));
+
+ svg.append("g")
+ .attr("class", "axis axis--y")
+ .call(d3.axisLeft().scale(y));
+
+ svg.append("text")
+ .attr("x", (WIDTH / 2))
+ .attr("y", 0 - (MARGIN.top / 2))
+ .attr("text-anchor", "middle")
+ .attr("class", "graph-title")
+ .text(graph.title);
}
- var x = d3.scaleTime()
- .domain([data_subset[0].date, data_subset[data_subset.length-1].date])
- .range([0, WIDTH]);
-
- var y = d3.scaleLinear()
- .domain([0, max])
- .range([HEIGHT, 0]);
-
- var stack = d3.stack()
- .keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"])
- .order(d3.stackOrderNone)
- .offset(d3.stackOffsetNone);
-
- var area = d3.area()
- .x(function(d, i) { return x(d.data.date); })
- .y0(function(d) { return y(d[0]); })
- .y1(function(d) { return y(d[1]); });
-
- var svg = d3.select("#" + graph.div).append("svg")
- .attr("width", WIDTH + MARGIN.left + MARGIN.right)
- .attr("height", HEIGHT + MARGIN.top + MARGIN.bottom)
- .append("g")
- .attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")");
-
- var layer = svg.selectAll(".layer")
- .data(stack(data_subset))
- .enter().append("g")
- //.attr("class", "layer");
-
- layer.append("path")
- //.attr("class", "area")
- .attr("class", function(d) { return key_to_color(d.key); })
- .attr("d", area);
-
- svg.append("g")
- .attr("class", "axis axis--x")
- .attr("transform", "translate(0," + HEIGHT + ")")
- .call(d3.axisBottom().scale(x));
-
- svg.append("g")
- .attr("class", "axis axis--y")
- .call(d3.axisLeft().scale(y));
-
- svg.append("text")
- .attr("x", (WIDTH / 2))
- .attr("y", 0 - (MARGIN.top / 2))
- .attr("text-anchor", "middle")
- .attr("class", "graph-title")
- .text(graph.title);
- }
-
-
- fallbackdirs_done = true;
- if(relays_done) {
- var toShow = document.getElementsByClassName('graphbox');
- for(i=0; i<toShow.length; i++) {
- toShow[i].style.display = 'block';
- }
- var toHide = document.getElementsByClassName('graphplaceholder');
- for(i=0; i<toHide.length; i++) {
- toHide[i].style.display = 'none';
+
+ fallbackdirs_done = true;
+ if(relays_done) {
+ var toShow = document.getElementsByClassName('graphbox');
+ for(i=0; i<toShow.length; i++) {
+ toShow[i].style.display = 'block';
+ }
+ var toHide = document.getElementsByClassName('graphplaceholder');
+ for(i=0; i<toHide.length; i++) {
+ toHide[i].style.display = 'none';
+ }
}
- }
- });
+ });
+ }
</script>"""
self.site.write(s)
@@ -654,9 +659,8 @@ if __name__ == '__main__':
CONFIG = stem.util.conf.config_dict('consensus', {
- 'ignored_authorities': [],
- 'bandwidth_authorities': [],
'known_params': [],
+ 'ignore_fallback_authorities': False,
})
config = stem.util.conf.get_config("consensus")
config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg'))
diff --git a/utility.py b/utility.py
new file mode 100644
index 0000000..6cebe6b
--- /dev/null
+++ b/utility.py
@@ -0,0 +1,86 @@
+import time
+import datetime
+
+import stem.descriptor
+import stem.descriptor.remote
+import stem.util.conf
+import stem.util.enum
+
+from stem.util.lru_cache import lru_cache
+
+ at lru_cache()
+def get_dirauths():
+ #Remove any BridgeAuths
+ return dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.v3ident)
+
+ at lru_cache()
+def get_bwauths():
+ return dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.is_bandwidth_authority)
+
+downloader = stem.descriptor.remote.DescriptorDownloader(
+ timeout = 60,
+ fall_back_to_authority = False,
+ document_handler = stem.descriptor.DocumentHandler.DOCUMENT,
+)
+
+def get_consensuses():
+ """
+ Provides a mapping of directory authority nicknames to their present consensus.
+
+ :returns: tuple of the form ({authority => consensus}, issues, runtimes)
+ """
+
+ return _get_documents('consensus', '/tor/status-vote/current/consensus')
+
+
+def get_votes():
+ """
+ Provides a mapping of directory authority nicknames to their present vote.
+
+ :returns: tuple of the form ({authority => vote}, issues, runtimes)
+ """
+
+ return _get_documents('vote', '/tor/status-vote/current/authority')
+
+
+def _get_documents(label, resource):
+ documents, issues, runtimes = {}, [], {}
+
+ for (nickname, authority) in get_dirauths().items():
+ if authority.v3ident is None:
+ continue # not a voting authority
+
+ query = downloader.query(
+ resource,
+ endpoints = [(authority.address, authority.dir_port)],
+ default_params = False,
+ )
+
+ try:
+ start_time = time.time()
+ documents[nickname] = query.run()[0]
+ runtimes[nickname] = time.time() - start_time
+ except Exception, exc:
+ if label == 'vote':
+ # try to download the vote via the other authorities
+
+ v3ident = authority.v3ident
+
+ query = downloader.query(
+ '/tor/status-vote/current/%s' % v3ident,
+ default_params = False,
+ )
+
+ query.run(True)
+
+ if not query.error:
+ documents[nickname] = list(query)[0]
+ continue
+
+ issues.append(('AUTHORITY_UNAVAILABLE', label, authority, query.download_url, exc))
+
+ return documents, issues, runtimes
+
+def unix_time(dt):
+ return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0
+
diff --git a/website.py b/website.py
index 7b79ec9..592ad0e 100755
--- a/website.py
+++ b/website.py
@@ -10,20 +10,23 @@ import os
import time
import operator
import datetime
-import stem.descriptor.remote
+
from base64 import b64decode
from Crypto.PublicKey import RSA
+import stem.descriptor.remote
+
+from utility import get_dirauths, get_bwauths
+
class WebsiteWriter:
consensus = None
votes = None
fallback_dirs = None
- config_set = False
known_authorities = []
- historical_bridge_authorities = []
bandwidth_authorities = []
consensus_expirey = datetime.timedelta(hours=3)
directory_key_warning_time = datetime.timedelta(days=14)
+ config = {}
known_params = []
def write_website(self, filename, include_relay_info=True):
self.site = open(filename, 'w')
@@ -49,13 +52,10 @@ class WebsiteWriter:
self.site.close()
def set_consensuses(self, c):
- if not self.config_set:
- raise Exception("Set config before calling")
self.consensuses = c
self.consensus = max(c.itervalues(), key=operator.attrgetter('valid_after'))
- self.known_authorities = set([r.nickname for r in self.consensus.routers.values() if 'Authority' in r.flags and r.nickname not in self.historical_bridge_authorities])
- self.known_authorities.update([r.nickname for r in self.consensus.directory_authorities])
- self.known_authorities.update([r for r in stem.descriptor.remote.get_authorities().keys() if r not in self.historical_bridge_authorities])
+ self.known_authorities = get_dirauths().keys()
+ self.bandwidth_authorities = get_bwauths().keys()
def set_votes(self, v):
self.votes = v
def set_consensus_expirey(self, timedelta):
@@ -63,10 +63,8 @@ class WebsiteWriter:
def set_directory_key_warning_time(self, timedelta):
self.directory_key_warning_time = timedelta
def set_config(self, config):
- self.config_set = True
+ self.config = config
self.known_params = config['known_params']
- self.historical_bridge_authorities = config['historical_bridge_authorities']
- self.bandwidth_authorities = config['bandwidth_authorities']
def set_fallback_dirs(self, fallback_dirs):
self.fallback_dirs = fallback_dirs
def get_consensus_time(self):
@@ -171,9 +169,9 @@ class WebsiteWriter:
+ " <td>" + dirauth_nickname + "</td>\n")
#Try and find a structure that has it's IP & Port
- authority = [r for r in self.consensus.routers.values() if r.nickname == dirauth_nickname and 'Authority' in r.flags]
+ authority = [r for r in self.consensus.routers.values() if r.nickname.lower() == dirauth_nickname and 'Authority' in r.flags]
if not authority:
- authority = [d for d in self.consensus.directory_authorities if d.nickname == dirauth_nickname]
+ authority = [d for d in self.consensus.directory_authorities if d.nickname.lower() == dirauth_nickname]
if authority:
authority = authority[0]
self.site.write(" <td><a href=\"http://" + authority.address + ":" + str(authority.dir_port)
@@ -183,20 +181,20 @@ class WebsiteWriter:
else:
self.site.write(" <td colspan=\"2\" class=\"oiv\">Missing entirely from consensus</td>\n")
- if dirauth_nickname in [d.nickname for d in self.consensus.directory_authorities]:
+ if dirauth_nickname in [d.nickname.lower() for d in self.consensus.directory_authorities]:
#The above structure is sufficient for getting the address & port
# but we need this structure for the authority's fingerprint
- authority = [d for d in self.consensus.directory_authorities if d.nickname == dirauth_nickname][0]
+ authority = [d for d in self.consensus.directory_authorities if d.nickname.lower() == dirauth_nickname][0]
if authority.fingerprint in signingFPs:
self.site.write(" <td>" + signingFPs[authority.fingerprint] + "</td>\n")
- elif authority.nickname in self.consensuses:
+ elif authority.nickname.lower() in self.consensuses:
self.site.write(" <td class=\"oiv\">Missing Signature! "
+ "Valid-after time of auth's displayed consensus: "
- + self.consensuses[authority.nickname].valid_after.isoformat().replace("T", " ")
+ + self.consensuses[authority.nickname.lower()].valid_after.isoformat().replace("T", " ")
+ "</td>\n")
else:
self.site.write(" <td class=\"oiv\">Missing Signature, and "
- + authority.nickname + " does not have a consensus available</td>\n")
+ + authority.nickname.lower() + " does not have a consensus available</td>\n")
self.site.write(" </tr>\n")
self.site.write("</table>\n")
@@ -592,6 +590,9 @@ class WebsiteWriter:
"""
Write the status of the fallback directory mirrors
"""
+ if self.config['ignore_fallback_authorities']:
+ return
+
self.site.write("<br>\n\n\n"
+ " <!-- ================================================================= -->"
+ "<a name=\"fallbackdirstatus\">\n"
@@ -649,7 +650,7 @@ class WebsiteWriter:
+ "Authority versions</a></h3>\n"
+ "<br>\n")
- authorityVersions = [(r.nickname, r.version) for r in self.consensus.routers.values() if 'Authority' in r.flags]
+ authorityVersions = [(r.nickname.lower(), r.version) for r in self.consensus.routers.values() if 'Authority' in r.flags]
if not authorityVersions:
self.site.write("<p>(No relays with Authority flag found.)</p>\n")
else:
@@ -683,6 +684,7 @@ class WebsiteWriter:
downloadData = {}
for l in lines:
parts = l.split(',')
+ parts[0] = parts[0].lower()
if int(parts[1]) < cutoff:
continue
if parts[0] not in downloadData:
@@ -1071,9 +1073,8 @@ if __name__ == '__main__':
CONFIG = stem.util.conf.config_dict('consensus', {
- 'ignored_authorities': [],
- 'bandwidth_authorities': [],
'known_params': [],
+ 'ignore_fallback_authorities': False,
})
config = stem.util.conf.get_config("consensus")
config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg'))
diff --git a/write_website.py b/write_website.py
index 7216d03..3d4760d 100755
--- a/write_website.py
+++ b/write_website.py
@@ -20,36 +20,35 @@ import stem.descriptor.remote
import stem.util.conf
import stem.util.enum
-from stem import Flag
-from stem.util.lru_cache import lru_cache
+from stem.descriptor.remote import FallbackDirectory
+from stem.descriptor.remote import DirectoryAuthority
+from utility import *
from website import WebsiteWriter
from graphs import GraphWriter
-from parseOldConsensuses import get_dirauths_in_tables
-DIRECTORY_AUTHORITIES = stem.descriptor.remote.get_authorities()
+
+#If you're running your own test network, you define your DirAuths here
+# dir-source line: dir-source authority_name v3ident hostname ip DirPort OrPort
+# r line: r nickname base64(fingerprint + "=") -> python -c "x = ''; import sys; import base64; sys.stdout.write(''.join('{:02x}'.format(ord(c)) for c in base64.b64decode(x)))"
+
+#stem.descriptor.remote.DIRECTORY_AUTHORITIES = {
+#'Faravahar': DirectoryAuthority(
+# nickname = 'Faravahar',
+# address = '154.35.175.225',
+# or_port = 443,
+# dir_port = 80,
+# is_bandwidth_authority = True,
+# fingerprint = 'CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC',
+# v3ident = 'EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97',
+# ),
+#}
CONFIG = stem.util.conf.config_dict('consensus', {
- 'ignored_authorities': [],
- 'bandwidth_authorities': [],
'known_params': [],
- 'historical_bridge_authorities' : []
+ 'ignore_fallback_authorities' : False
})
-downloader = stem.descriptor.remote.DescriptorDownloader(
- timeout = 60,
- fall_back_to_authority = False,
- document_handler = stem.descriptor.DocumentHandler.DOCUMENT,
-)
-
-
- at lru_cache()
-def directory_authorities():
- return dict((k, v) for (k, v) in DIRECTORY_AUTHORITIES.items() if k not in CONFIG['ignored_authorities'])
-
-def unix_time(dt):
- return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0
-
def main():
# loads configuration data
config = stem.util.conf.get_config("consensus")
@@ -65,47 +64,54 @@ def main():
f.close()
# Calculate the fallback directory info
- from stem.descriptor.remote import FallbackDirectory
- fallback_dirs = stem.descriptor.remote.FallbackDirectory.from_remote()
+ if not CONFIG['ignore_fallback_authorities']:
+ fallback_dirs = stem.descriptor.remote.FallbackDirectory.from_remote()
+ else:
+ fallback_dirs = []
+
# great for debugging
#import pickle
+ #pickle.dump(consensuses, open('consensus.p', 'wb'))
+ #pickle.dump(votes, open('votes.p', 'wb'))
#pickle.dump(fallback_dirs, open('fallback_dirs.p', 'wb'))
- # Calculate the number of fallback directory authorities present in the consensus and insert it into the database
- fallback_dirs_running = 0
- fallback_dirs_notrunning = 0
- for relay_fp in consensuses.values()[0].routers:
- if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags:
- fallback_dirs_running += 1
- elif relay_fp in fallback_dirs:
- fallback_dirs_notrunning += 1
-
- insertValues = [unix_time(consensuses.values()[0].valid_after)]
- insertValues.append(fallback_dirs_running)
- insertValues.append(fallback_dirs_notrunning)
- insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning)
dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
- dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));")
- dbc.commit()
+ # Calculate the number of fallback directory authorities present in the consensus and insert it into the database
+ if not CONFIG['ignore_fallback_authorities']:
+ fallback_dirs_running = 0
+ fallback_dirs_notrunning = 0
+ for relay_fp in consensuses.values()[0].routers:
+ if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags:
+ fallback_dirs_running += 1
+ elif relay_fp in fallback_dirs:
+ fallback_dirs_notrunning += 1
+
+ insertValues = [unix_time(consensuses.values()[0].valid_after)]
+ insertValues.append(fallback_dirs_running)
+ insertValues.append(fallback_dirs_notrunning)
+ insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning)
+
+ dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));")
+ dbc.commit()
- dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues)
- dbc.commit()
+ dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues)
+ dbc.commit()
- # Write out the updated csv file for the graphs
- fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160")
- f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w')
- f.write("date")
- f.write(",fallback_dirs_running")
- f.write(",fallback_dirs_notrunning")
- f.write(",fallback_dirs_missing")
- f.write("\n")
- for r in fallback_dir_data.fetchall():
- for v in r:
- f.write(("0" if v == None else str(v)) + ",")
+ # Write out the updated csv file for the graphs
+ fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160")
+ f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w')
+ f.write("date")
+ f.write(",fallback_dirs_running")
+ f.write(",fallback_dirs_notrunning")
+ f.write(",fallback_dirs_missing")
f.write("\n")
- f.close()
+ for r in fallback_dir_data.fetchall():
+ for v in r:
+ f.write(("0" if v == None else str(v)) + ",")
+ f.write("\n")
+ f.close()
# Calculate the number of known and measured relays for each dirauth and insert it into the database
data = {}
@@ -130,10 +136,8 @@ def main():
createColumns = ""
insertColumns = "date"
insertQuestions = ""
- import pdb
- pdb.set_trace()
- for dirauth_nickname in directory_authorities():
- dirauth_nickname = dirauth_nickname.lower()
+
+ for dirauth_nickname in get_dirauths():
if vote_data_columns and dirauth_nickname not in vote_data_columns:
dbc.execute("ALTER TABLE vote_data ADD COLUMN " + dirauth_nickname + "_known integer")
dbc.execute("ALTER TABLE vote_data ADD COLUMN " + dirauth_nickname + "_running integer")
@@ -155,12 +159,15 @@ def main():
dbc.commit()
# Write out the updated csv file for the graphs
+ vote_data_columns = []
+ vote_data_schema = dbc.execute("PRAGMA table_info(vote_data)")
+ for c in vote_data_schema:
+ vote_data_columns.append(c[1])
+
vote_data = dbc.execute("SELECT * from vote_data ORDER BY date DESC LIMIT 2160")
f = open(os.path.join(os.path.dirname(__file__), 'out', 'vote-stats.csv'), 'w')
- f.write("date")
- for d in get_dirauths_in_tables():
- s = "," + d + "_known," + d + "_running," + d + "_bwauth"
- f.write(s)
+ for c in vote_data_columns:
+ f.write(c + ",")
f.write("\n")
for r in vote_data.fetchall():
for v in r:
@@ -168,11 +175,6 @@ def main():
f.write("\n")
f.close()
- # great for debugging
- #import pickle
- #pickle.dump(consensuses, open('consensus.p', 'wb'))
- #pickle.dump(votes, open('votes.p', 'wb'))
-
# produces the website
w = WebsiteWriter()
w.set_config(CONFIG)
@@ -220,65 +222,6 @@ def main():
os.remove(os.path.join(os.path.dirname(__file__), 'out', f))
-def get_consensuses():
- """
- Provides a mapping of directory authority nicknames to their present consensus.
-
- :returns: tuple of the form ({authority => consensus}, issues, runtimes)
- """
-
- return _get_documents('consensus', '/tor/status-vote/current/consensus')
-
-
-def get_votes():
- """
- Provides a mapping of directory authority nicknames to their present vote.
-
- :returns: tuple of the form ({authority => vote}, issues, runtimes)
- """
-
- return _get_documents('vote', '/tor/status-vote/current/authority')
-
-
-def _get_documents(label, resource):
- documents, issues, runtimes = {}, [], {}
-
- for authority in directory_authorities().values():
- if authority.v3ident is None:
- continue # not a voting authority
-
- query = downloader.query(
- resource,
- endpoints = [(authority.address, authority.dir_port)],
- default_params = False,
- )
-
- try:
- start_time = time.time()
- documents[authority.nickname] = query.run()[0]
- runtimes[authority.nickname] = time.time() - start_time
- except Exception, exc:
- if label == 'vote':
- # try to download the vote via the other authorities
-
- v3ident = directory_authorities()[authority.nickname].v3ident
-
- query = downloader.query(
- '/tor/status-vote/current/%s' % v3ident,
- default_params = False,
- )
-
- query.run(True)
-
- if not query.error:
- documents[authority.nickname] = list(query)[0]
- continue
-
- issues.append(('AUTHORITY_UNAVAILABLE', label, authority, query.download_url, exc))
-
- return documents, issues, runtimes
-
-
if __name__ == '__main__':
try:
main()
More information about the tor-commits
mailing list