commit b30311728584e0682c3d02079932611d3d547e1a Author: Tom tom@ritter.vg Date: Thu Oct 27 14:34:41 2016 -0500

Big refactor

- Put in a stub for putting your own DirAuths in - Fix 20480 by lowercasing dirauth names consistenty - Fix 20464 by making all operations with vote_data smarter. - Insert with named columns - Examine table structure and add columns if a dirauth gets added - select named columns for the csv - Remove last hardcoded dirauth stuff from graphs and make the colors dynamic - Add a config option to ignore fallback directories (important for other networks) - Remove super-redundant dirauth searching in website.py - Move some stuff to a utility file --- .gitignore | 4 + data/consensus.cfg | 17 +-- graphs.py | 380 +++++++++++++++++++++++++++-------------------------- utility.py | 86 ++++++++++++ website.py | 43 +++--- write_website.py | 191 ++++++++++----------------- 6 files changed, 373 insertions(+), 348 deletions(-)

diff --git a/.gitignore b/.gitignore index 45104d6..037b253 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,13 @@ *.p *.swp stem + out/index.html out/graphs.html out/vote-stats.csv out/consensus-health* out/download-stats.csv +out/fallback-dir-stats.csv +out/historical.db + data/historical.db \ No newline at end of file diff --git a/data/consensus.cfg b/data/consensus.cfg index c4a2142..381684a 100644 --- a/data/consensus.cfg +++ b/data/consensus.cfg @@ -1,19 +1,6 @@ -# directory authorities for which to not provide data +# control consensus-health output

-ignored_authorities turtles - -# directory authorities that run bandwidth scanners - -bandwidth_authorities moria1 -bandwidth_authorities gabelmoo -bandwidth_authorities tor26 -bandwidth_authorities longclaw -bandwidth_authorities maatuska - -# all current/historical bridge auths - -historical_bridge_authorities tonga -historical_bridge_authorities bifrost +ignore_fallback_authorities False

# recognized tor consensus parameters

diff --git a/graphs.py b/graphs.py index 9293b1f..6738a52 100755 --- a/graphs.py +++ b/graphs.py @@ -14,7 +14,7 @@ import stem.descriptor.remote from base64 import b64decode

from website import WebsiteWriter -from parseOldConsensuses import get_dirauths_in_tables +from utility import get_dirauths, get_bwauths

class GraphWriter(WebsiteWriter): def write_website(self, filename): @@ -65,13 +65,14 @@ class GraphWriter(WebsiteWriter): + " text-align: center;\n" + " display: none;\n" + " }\n" - + " .faravahar {\n" + + + " .auth1 {\n" + " fill: none;\n" + " stroke: steelblue;\n" + " background-color: steelblue;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .gabelmoo, .orange {\n" + + " .auth2, .orange {\n" + " fill: none;\n" + " stroke: orange;\n" + " background-color: orange;\n" @@ -80,13 +81,13 @@ class GraphWriter(WebsiteWriter): + " .orange {\n" + " fill: orange;" + " }\n" - + " .moria1 {\n" + + " .auth3 {\n" + " fill: none;\n" + " stroke: yellow;\n" + " background-color: yellow;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .maatuska, .green {\n" + + " .auth4, .green {\n" + " fill: none;\n" + " stroke: green;\n" + " background-color: green;\n" @@ -95,7 +96,7 @@ class GraphWriter(WebsiteWriter): + " .green {\n" + " fill: green;" + " }\n" - + " .longclaw, .red {\n" + + " .auth5, .red {\n" + " fill: none;\n" + " stroke: red;\n" + " background-color: red;\n" @@ -104,36 +105,37 @@ class GraphWriter(WebsiteWriter): + " .red {\n" + " fill: red;" + " }\n" - + " .tor26 {\n" + + " .auth6 {\n" + " fill: none;\n" + " stroke: purple;\n" + " background-color: purple;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .urras {\n" + + " .auth7 {\n" + " fill: none;\n" + " stroke: black;\n" + " background-color: black;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .turtles {\n" + + " .auth8 {\n" + " fill: none;\n" + " stroke: #0000FF;\n" + " background-color: #0000FF;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .dizum {\n" + + " .auth9 {\n" + " fill: none;\n" + " stroke: limegreen;\n" + " background-color: limegreen;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .dannenberg {\n" + + " .auth10 {\n" + " fill: none;\n" + " stroke: pink;\n" + " background-color: pink;\n" + " stroke-width: 1.5px;\n" + " }\n" + + " </style>\n" + " <div class="center">\n" + " <div class="main-column">\n" @@ -159,6 +161,9 @@ class GraphWriter(WebsiteWriter): """ Write the graphs of the fallback directory mirrors """ + if self.config['ignore_fallback_authorities']: + return + self.site.write("<br>\n\n\n" + " <!-- ================================================================= -->" + "<a name="fallbackdirgraphs">\n" @@ -167,8 +172,7 @@ class GraphWriter(WebsiteWriter): + "<br>\n" + "<table border="0" cellpadding="4" cellspacing="0" summary="">\n" + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" + + " <col width="800">\n" + " </colgroup>\n" + " <tr class="graphplaceholder">\n" + " <td>\n" @@ -185,19 +189,17 @@ class GraphWriter(WebsiteWriter): self.site.write("</table>\n")

#----------------------------------------------------------------------------------------- - def _write_number_of_relays_voted_about_graphs_spot(self, divName): + def _write_number_of_relays_voted_about_graphs_spot(self, divName, dirAuths): self.site.write(" <tr>\n" + " <td>\n" - + " <div id="" + str(divName) + "" class="graphbox">\n" - + " <span class="moria1" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Moria\n" - + " <span class="faravahar" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Faravahar\n" - + " <span class="gabelmoo" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Gabelmoo\n" - + " <span class="maatuska" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Maatuska\n" - + " <span class="longclaw" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Longclaw\n" - + " <span class="tor26" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> tor26\n" - + " <span class="dizum" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> dizum\n" - + " <span class="dannenberg" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> dannenberg\n" - + " </div>\n" + + " <div id="" + str(divName) + "" class="graphbox">\n") + + i = 0 + for d in dirAuths: + i += 1 + self.site.write(" <span class="auth" + str(i) + "" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> " + d + "\n") + + self.site.write(" </div>\n" + " </td>\n" + " </tr>\n") def _write_number_of_relays_voted_about_graphs(self): @@ -212,8 +214,7 @@ class GraphWriter(WebsiteWriter): + "<br>\n" + "<table border="0" cellpadding="4" cellspacing="0" summary="">\n" + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" + + " <col width="800">\n" + " </colgroup>\n" + " <tr class="graphplaceholder">\n" + " <td>\n" @@ -222,31 +223,32 @@ class GraphWriter(WebsiteWriter): + " </div>\n" + " </td>\n" + " </tr>\n") - self._write_number_of_relays_voted_about_graphs_spot("voted_total_1") - self._write_number_of_relays_voted_about_graphs_spot("voted_total_2") - self._write_number_of_relays_voted_about_graphs_spot("voted_total_3") - self._write_number_of_relays_voted_about_graphs_spot("voted_total_4") - self._write_number_of_relays_voted_about_graphs_spot("voted_running_1") - self._write_number_of_relays_voted_about_graphs_spot("voted_running_2") - self._write_number_of_relays_voted_about_graphs_spot("voted_running_3") - self._write_number_of_relays_voted_about_graphs_spot("voted_running_4") - self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_1") - self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_2") - self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_3") - self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_4") + self._write_number_of_relays_voted_about_graphs_spot("voted_total_1", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_total_2", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_total_3", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_total_4", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_running_1", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_running_2", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_running_3", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_running_4", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_1", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_2", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_3", get_dirauths()) + self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_4", get_dirauths()) self.site.write("</table>\n")

#----------------------------------------------------------------------------------------- - def _write_bandwidth_scanner_graphs_spot(self, divName): + def _write_bandwidth_scanner_graphs_spot(self, divName, bwAuths): self.site.write(" <tr>\n" + " <td>\n" - + " <div id="" + str(divName) + "" class="graphbox">\n" - + " <span class="moria1" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Moria\n" - + " <span class="faravahar" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Faravahar\n" - + " <span class="gabelmoo" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Gabelmoo\n" - + " <span class="maatuska" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Maatuska\n" - + " <span class="longclaw" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Longclaw\n" - + " </div>\n" + + " <div id="" + str(divName) + "" class="graphbox">\n") + + i = 0 + for d in bwAuths: + i += 1 + self.site.write(" <span class="auth" + str(i) + "" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> " + d + "\n") + + self.site.write(" </div>\n" + " </td>\n" + " </tr>\n") def _write_bandwidth_scanner_graphs(self): @@ -261,8 +263,7 @@ class GraphWriter(WebsiteWriter): + "<br>\n" + "<table border="0" cellpadding="4" cellspacing="0" summary="">\n" + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" + + " <col width="800">\n" + " </colgroup>\n" + " <tr class="graphplaceholder">\n" + " <td>\n" @@ -271,10 +272,10 @@ class GraphWriter(WebsiteWriter): + " </div>\n" + " </td>\n" + " </tr>\n") - self._write_bandwidth_scanner_graphs_spot("bwauth_measured_1") - self._write_bandwidth_scanner_graphs_spot("bwauth_measured_2") - self._write_bandwidth_scanner_graphs_spot("bwauth_measured_3") - self._write_bandwidth_scanner_graphs_spot("bwauth_measured_4") + self._write_bandwidth_scanner_graphs_spot("bwauth_measured_1", get_bwauths()) + self._write_bandwidth_scanner_graphs_spot("bwauth_measured_2", get_bwauths()) + self._write_bandwidth_scanner_graphs_spot("bwauth_measured_3", get_bwauths()) + self._write_bandwidth_scanner_graphs_spot("bwauth_measured_4", get_bwauths()) #self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_1") #self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_2") #self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_3") @@ -289,10 +290,9 @@ class GraphWriter(WebsiteWriter): HEIGHT = 500, MARGIN = {top: 40, right: 40, bottom: 40, left: 40};

- var bwauths = ["faravahar","gabelmoo","moria1","maatuska","longclaw"]; - var dirauths = """ + str(get_dirauths_in_tables()) + """; - dirauths.splice(dirauths.indexOf('urras'), 1); - dirauths.splice(dirauths.indexOf('turtles'), 1); + var bwauths = """ + str(get_bwauths().keys()) + """; + var dirauths = """ + str(get_dirauths().keys()) + """; + var ignore_fallback_dirs = """ + str(self.config['ignore_fallback_authorities']).lower() + """;

var _getBandwidthDataValue = function(d, dirauth) { return d[dirauth + "_bwauth"]; } var _getRunningDataValue = function(d, dirauth) { return d[dirauth + "_running"]; } @@ -360,7 +360,7 @@ class GraphWriter(WebsiteWriter): ];

relays_done = false; - fallbackdirs_done = false; + fallbackdirs_done = ignore_fallback_dirs; fetch("vote-stats.csv").then(function(response) { return response.text(); }).then(function(text) { @@ -381,11 +381,11 @@ class GraphWriter(WebsiteWriter): graph = GRAPHS_TO_GENERATE[g];

if(graph.data_slice+1 > data.length) { - data_subset = data.slice(1); + data_subset = data.slice(0); console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items..."); } else - data_subset = data.slice(1, graph.data_slice+1); + data_subset = data.slice(0, graph.data_slice+1); data_subset.reverse();

// Calculate the Graph Boundaries ----------------------------------------- @@ -436,11 +436,12 @@ class GraphWriter(WebsiteWriter): .domain([avg-(5*stddev), avg+(5*stddev)]) .range([HEIGHT, 0]);

+ var i = 1; var lines = [] for(auth in graph.authorities) { this_auth = graph.authorities[auth]; - lines.push({auth: this_auth, line: (function(dirAuthClosure) { + lines.push({authIndex: i, line: (function(dirAuthClosure) { return d3.line() .defined(function(d) { return d && graph.data_func(d, dirAuthClosure) && @@ -449,6 +450,7 @@ class GraphWriter(WebsiteWriter): .x(function(d) { return x(d.date); }) .y(function(d) { return y(graph.data_func(d, dirAuthClosure)); }); })(this_auth)}); + i++; }

var svg = d3.select("#" + graph.div).append("svg") @@ -470,7 +472,7 @@ class GraphWriter(WebsiteWriter): for(l in lines) { svg.append("path") - .attr("class", lines[l].auth) + .attr("class", "auth" + lines[l].authIndex) .attr("d", lines[l].line); }

@@ -496,136 +498,139 @@ class GraphWriter(WebsiteWriter):

});

- fetch("fallback-dir-stats.csv").then(function(response) { - return response.text(); - }).then(function(text) { - return d3.csvParse(text, function(d) { - for(i in d) { - if(i == "date") - d[i] = new Date(Number(d[i])); + if(!ignore_fallback_dirs) { + + fetch("fallback-dir-stats.csv").then(function(response) { + return response.text(); + }).then(function(text) { + return d3.csvParse(text, function(d) { + for(i in d) { + if(i == "date") + d[i] = new Date(Number(d[i])); + else + d[i] = Number(d[i]); + } + return d; + }); + }).then(function(data) { + var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' }; + /*Pie Graph + data_subset = data.slice(0); + data_subset = [ + {'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']}, + {'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']}, + {'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']}, + ]; + var data_func = function(d) { return d.value; }; + var arcs = d3.pie() + .sort(null) + .value(data_func)(data_subset); + + var svg = d3.select('#fallbackdirs_pie') + .append('svg') + .attr('width', WIDTH) + .attr('height', HEIGHT) + .append('g') + .attr('transform', 'translate(' + (WIDTH / 2) + ',' + (HEIGHT / 2) + ')'); + + var arc = d3.arc() + .innerRadius(0) + .outerRadius(100); + + var path = svg.selectAll('path') + .data(arcs) + .enter() + .append('path') + .attr('d', arc) + .attr('class', function(d, i) { + return key_to_color(d.data.label); + });*/ + + //Line Graphs + for(g in FALLBACK_GRAPHS_TO_GENERATE) + { + graph = FALLBACK_GRAPHS_TO_GENERATE[g]; + + if(graph.data_slice+1 > data.length) { + data_subset = data.slice(0); + console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items..."); + } else - d[i] = Number(d[i]); - } - return d; - }); - }).then(function(data) { - var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' }; - /*Pie Graph - data_subset = data.slice(0); - data_subset = [ - {'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']}, - {'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']}, - {'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']}, - ]; - var data_func = function(d) { return d.value; }; - var arcs = d3.pie() - .sort(null) - .value(data_func)(data_subset); - - var svg = d3.select('#fallbackdirs_pie') - .append('svg') - .attr('width', WIDTH) - .attr('height', HEIGHT) - .append('g') - .attr('transform', 'translate(' + (WIDTH / 2) + ',' + (HEIGHT / 2) + ')'); - - var arc = d3.arc() - .innerRadius(0) - .outerRadius(100); - - var path = svg.selectAll('path') - .data(arcs) - .enter() - .append('path') - .attr('d', arc) - .attr('class', function(d, i) { - return key_to_color(d.data.label); - });*/ - - //Line Graphs - for(g in FALLBACK_GRAPHS_TO_GENERATE) - { - graph = FALLBACK_GRAPHS_TO_GENERATE[g]; + data_subset = data.slice(0, graph.data_slice); + data_subset.reverse(); + + max = 0 + for(d in data_subset) { + x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing']; + if(x > max) + max = x; + }

- if(graph.data_slice+1 > data.length) { - data_subset = data.slice(0); - console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items..."); - } - else - data_subset = data.slice(0, graph.data_slice); - data_subset.reverse(); - - max = 0 - for(d in data_subset) { - x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing']; - if(x > max) - max = x; + var x = d3.scaleTime() + .domain([data_subset[0].date, data_subset[data_subset.length-1].date]) + .range([0, WIDTH]); + + var y = d3.scaleLinear() + .domain([0, max]) + .range([HEIGHT, 0]); + + var stack = d3.stack() + .keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"]) + .order(d3.stackOrderNone) + .offset(d3.stackOffsetNone); + + var area = d3.area() + .x(function(d, i) { return x(d.data.date); }) + .y0(function(d) { return y(d[0]); }) + .y1(function(d) { return y(d[1]); }); + + var svg = d3.select("#" + graph.div).append("svg") + .attr("width", WIDTH + MARGIN.left + MARGIN.right) + .attr("height", HEIGHT + MARGIN.top + MARGIN.bottom) + .append("g") + .attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")"); + + var layer = svg.selectAll(".layer") + .data(stack(data_subset)) + .enter().append("g") + //.attr("class", "layer"); + + layer.append("path") + //.attr("class", "area") + .attr("class", function(d) { return key_to_color(d.key); }) + .attr("d", area); + + svg.append("g") + .attr("class", "axis axis--x") + .attr("transform", "translate(0," + HEIGHT + ")") + .call(d3.axisBottom().scale(x)); + + svg.append("g") + .attr("class", "axis axis--y") + .call(d3.axisLeft().scale(y)); + + svg.append("text") + .attr("x", (WIDTH / 2)) + .attr("y", 0 - (MARGIN.top / 2)) + .attr("text-anchor", "middle") + .attr("class", "graph-title") + .text(graph.title); }

- var x = d3.scaleTime() - .domain([data_subset[0].date, data_subset[data_subset.length-1].date]) - .range([0, WIDTH]); - - var y = d3.scaleLinear() - .domain([0, max]) - .range([HEIGHT, 0]); - - var stack = d3.stack() - .keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"]) - .order(d3.stackOrderNone) - .offset(d3.stackOffsetNone); - - var area = d3.area() - .x(function(d, i) { return x(d.data.date); }) - .y0(function(d) { return y(d[0]); }) - .y1(function(d) { return y(d[1]); }); - - var svg = d3.select("#" + graph.div).append("svg") - .attr("width", WIDTH + MARGIN.left + MARGIN.right) - .attr("height", HEIGHT + MARGIN.top + MARGIN.bottom) - .append("g") - .attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")"); - - var layer = svg.selectAll(".layer") - .data(stack(data_subset)) - .enter().append("g") - //.attr("class", "layer"); - - layer.append("path") - //.attr("class", "area") - .attr("class", function(d) { return key_to_color(d.key); }) - .attr("d", area); - - svg.append("g") - .attr("class", "axis axis--x") - .attr("transform", "translate(0," + HEIGHT + ")") - .call(d3.axisBottom().scale(x)); - - svg.append("g") - .attr("class", "axis axis--y") - .call(d3.axisLeft().scale(y)); - - svg.append("text") - .attr("x", (WIDTH / 2)) - .attr("y", 0 - (MARGIN.top / 2)) - .attr("text-anchor", "middle") - .attr("class", "graph-title") - .text(graph.title); - } - - - fallbackdirs_done = true; - if(relays_done) { - var toShow = document.getElementsByClassName('graphbox'); - for(i=0; i<toShow.length; i++) { - toShow[i].style.display = 'block'; - } - var toHide = document.getElementsByClassName('graphplaceholder'); - for(i=0; i<toHide.length; i++) { - toHide[i].style.display = 'none'; + + fallbackdirs_done = true; + if(relays_done) { + var toShow = document.getElementsByClassName('graphbox'); + for(i=0; i<toShow.length; i++) { + toShow[i].style.display = 'block'; + } + var toHide = document.getElementsByClassName('graphplaceholder'); + for(i=0; i<toHide.length; i++) { + toHide[i].style.display = 'none'; + } } - } - }); + }); + }

</script>""" self.site.write(s) @@ -654,9 +659,8 @@ if __name__ == '__main__':

CONFIG = stem.util.conf.config_dict('consensus', { - 'ignored_authorities': [], - 'bandwidth_authorities': [], 'known_params': [], + 'ignore_fallback_authorities': False, }) config = stem.util.conf.get_config("consensus") config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg')) diff --git a/utility.py b/utility.py new file mode 100644 index 0000000..6cebe6b --- /dev/null +++ b/utility.py @@ -0,0 +1,86 @@ +import time +import datetime + +import stem.descriptor +import stem.descriptor.remote +import stem.util.conf +import stem.util.enum + +from stem.util.lru_cache import lru_cache + +@lru_cache() +def get_dirauths(): + #Remove any BridgeAuths + return dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.v3ident) + +@lru_cache() +def get_bwauths(): + return dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.is_bandwidth_authority) + +downloader = stem.descriptor.remote.DescriptorDownloader( + timeout = 60, + fall_back_to_authority = False, + document_handler = stem.descriptor.DocumentHandler.DOCUMENT, +) + +def get_consensuses(): + """ + Provides a mapping of directory authority nicknames to their present consensus. + + :returns: tuple of the form ({authority => consensus}, issues, runtimes) + """ + + return _get_documents('consensus', '/tor/status-vote/current/consensus') + + +def get_votes(): + """ + Provides a mapping of directory authority nicknames to their present vote. + + :returns: tuple of the form ({authority => vote}, issues, runtimes) + """ + + return _get_documents('vote', '/tor/status-vote/current/authority') + + +def _get_documents(label, resource): + documents, issues, runtimes = {}, [], {} + + for (nickname, authority) in get_dirauths().items(): + if authority.v3ident is None: + continue # not a voting authority + + query = downloader.query( + resource, + endpoints = [(authority.address, authority.dir_port)], + default_params = False, + ) + + try: + start_time = time.time() + documents[nickname] = query.run()[0] + runtimes[nickname] = time.time() - start_time + except Exception, exc: + if label == 'vote': + # try to download the vote via the other authorities + + v3ident = authority.v3ident + + query = downloader.query( + '/tor/status-vote/current/%s' % v3ident, + default_params = False, + ) + + query.run(True) + + if not query.error: + documents[nickname] = list(query)[0] + continue + + issues.append(('AUTHORITY_UNAVAILABLE', label, authority, query.download_url, exc)) + + return documents, issues, runtimes + +def unix_time(dt): + return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0 + diff --git a/website.py b/website.py index 7b79ec9..592ad0e 100755 --- a/website.py +++ b/website.py @@ -10,20 +10,23 @@ import os import time import operator import datetime -import stem.descriptor.remote + from base64 import b64decode from Crypto.PublicKey import RSA

+import stem.descriptor.remote + +from utility import get_dirauths, get_bwauths + class WebsiteWriter: consensus = None votes = None fallback_dirs = None - config_set = False known_authorities = [] - historical_bridge_authorities = [] bandwidth_authorities = [] consensus_expirey = datetime.timedelta(hours=3) directory_key_warning_time = datetime.timedelta(days=14) + config = {} known_params = [] def write_website(self, filename, include_relay_info=True): self.site = open(filename, 'w') @@ -49,13 +52,10 @@ class WebsiteWriter: self.site.close()

def set_consensuses(self, c): - if not self.config_set: - raise Exception("Set config before calling") self.consensuses = c self.consensus = max(c.itervalues(), key=operator.attrgetter('valid_after')) - self.known_authorities = set([r.nickname for r in self.consensus.routers.values() if 'Authority' in r.flags and r.nickname not in self.historical_bridge_authorities]) - self.known_authorities.update([r.nickname for r in self.consensus.directory_authorities]) - self.known_authorities.update([r for r in stem.descriptor.remote.get_authorities().keys() if r not in self.historical_bridge_authorities]) + self.known_authorities = get_dirauths().keys() + self.bandwidth_authorities = get_bwauths().keys() def set_votes(self, v): self.votes = v def set_consensus_expirey(self, timedelta): @@ -63,10 +63,8 @@ class WebsiteWriter: def set_directory_key_warning_time(self, timedelta): self.directory_key_warning_time = timedelta def set_config(self, config): - self.config_set = True + self.config = config self.known_params = config['known_params'] - self.historical_bridge_authorities = config['historical_bridge_authorities'] - self.bandwidth_authorities = config['bandwidth_authorities'] def set_fallback_dirs(self, fallback_dirs): self.fallback_dirs = fallback_dirs def get_consensus_time(self): @@ -171,9 +169,9 @@ class WebsiteWriter: + " <td>" + dirauth_nickname + "</td>\n") #Try and find a structure that has it's IP & Port - authority = [r for r in self.consensus.routers.values() if r.nickname == dirauth_nickname and 'Authority' in r.flags] + authority = [r for r in self.consensus.routers.values() if r.nickname.lower() == dirauth_nickname and 'Authority' in r.flags] if not authority: - authority = [d for d in self.consensus.directory_authorities if d.nickname == dirauth_nickname] + authority = [d for d in self.consensus.directory_authorities if d.nickname.lower() == dirauth_nickname] if authority: authority = authority[0] self.site.write(" <td><a href="http://" + authority.address + ":" + str(authority.dir_port) @@ -183,20 +181,20 @@ class WebsiteWriter: else: self.site.write(" <td colspan="2" class="oiv">Missing entirely from consensus</td>\n") - if dirauth_nickname in [d.nickname for d in self.consensus.directory_authorities]: + if dirauth_nickname in [d.nickname.lower() for d in self.consensus.directory_authorities]: #The above structure is sufficient for getting the address & port # but we need this structure for the authority's fingerprint - authority = [d for d in self.consensus.directory_authorities if d.nickname == dirauth_nickname][0] + authority = [d for d in self.consensus.directory_authorities if d.nickname.lower() == dirauth_nickname][0] if authority.fingerprint in signingFPs: self.site.write(" <td>" + signingFPs[authority.fingerprint] + "</td>\n") - elif authority.nickname in self.consensuses: + elif authority.nickname.lower() in self.consensuses: self.site.write(" <td class="oiv">Missing Signature! " + "Valid-after time of auth's displayed consensus: " - + self.consensuses[authority.nickname].valid_after.isoformat().replace("T", " ") + + self.consensuses[authority.nickname.lower()].valid_after.isoformat().replace("T", " ") + "</td>\n") else: self.site.write(" <td class="oiv">Missing Signature, and " - + authority.nickname + " does not have a consensus available</td>\n") + + authority.nickname.lower() + " does not have a consensus available</td>\n") self.site.write(" </tr>\n") self.site.write("</table>\n")

@@ -592,6 +590,9 @@ class WebsiteWriter: """ Write the status of the fallback directory mirrors """ + if self.config['ignore_fallback_authorities']: + return + self.site.write("<br>\n\n\n" + " <!-- ================================================================= -->" + "<a name="fallbackdirstatus">\n" @@ -649,7 +650,7 @@ class WebsiteWriter: + "Authority versions</a></h3>\n" + "<br>\n")

- authorityVersions = [(r.nickname, r.version) for r in self.consensus.routers.values() if 'Authority' in r.flags] + authorityVersions = [(r.nickname.lower(), r.version) for r in self.consensus.routers.values() if 'Authority' in r.flags] if not authorityVersions: self.site.write("<p>(No relays with Authority flag found.)</p>\n") else: @@ -683,6 +684,7 @@ class WebsiteWriter: downloadData = {} for l in lines: parts = l.split(',') + parts[0] = parts[0].lower() if int(parts[1]) < cutoff: continue if parts[0] not in downloadData: @@ -1071,9 +1073,8 @@ if __name__ == '__main__':

CONFIG = stem.util.conf.config_dict('consensus', { - 'ignored_authorities': [], - 'bandwidth_authorities': [], 'known_params': [], + 'ignore_fallback_authorities': False, }) config = stem.util.conf.get_config("consensus") config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg')) diff --git a/write_website.py b/write_website.py index 7216d03..3d4760d 100755 --- a/write_website.py +++ b/write_website.py @@ -20,36 +20,35 @@ import stem.descriptor.remote import stem.util.conf import stem.util.enum

-from stem import Flag -from stem.util.lru_cache import lru_cache +from stem.descriptor.remote import FallbackDirectory +from stem.descriptor.remote import DirectoryAuthority

+from utility import * from website import WebsiteWriter from graphs import GraphWriter -from parseOldConsensuses import get_dirauths_in_tables

-DIRECTORY_AUTHORITIES = stem.descriptor.remote.get_authorities() + +#If you're running your own test network, you define your DirAuths here +# dir-source line: dir-source authority_name v3ident hostname ip DirPort OrPort +# r line: r nickname base64(fingerprint + "=") -> python -c "x = ''; import sys; import base64; sys.stdout.write(''.join('{:02x}'.format(ord(c)) for c in base64.b64decode(x)))" + +#stem.descriptor.remote.DIRECTORY_AUTHORITIES = { +#'Faravahar': DirectoryAuthority( +# nickname = 'Faravahar', +# address = '154.35.175.225', +# or_port = 443, +# dir_port = 80, +# is_bandwidth_authority = True, +# fingerprint = 'CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC', +# v3ident = 'EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97', +# ), +#}

CONFIG = stem.util.conf.config_dict('consensus', { - 'ignored_authorities': [], - 'bandwidth_authorities': [], 'known_params': [], - 'historical_bridge_authorities' : [] + 'ignore_fallback_authorities' : False })

-downloader = stem.descriptor.remote.DescriptorDownloader( - timeout = 60, - fall_back_to_authority = False, - document_handler = stem.descriptor.DocumentHandler.DOCUMENT, -) - - -@lru_cache() -def directory_authorities(): - return dict((k, v) for (k, v) in DIRECTORY_AUTHORITIES.items() if k not in CONFIG['ignored_authorities']) - -def unix_time(dt): - return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0 - def main(): # loads configuration data config = stem.util.conf.get_config("consensus") @@ -65,47 +64,54 @@ def main(): f.close()

# Calculate the fallback directory info - from stem.descriptor.remote import FallbackDirectory - fallback_dirs = stem.descriptor.remote.FallbackDirectory.from_remote() + if not CONFIG['ignore_fallback_authorities']: + fallback_dirs = stem.descriptor.remote.FallbackDirectory.from_remote() + else: + fallback_dirs = [] + # great for debugging #import pickle + #pickle.dump(consensuses, open('consensus.p', 'wb')) + #pickle.dump(votes, open('votes.p', 'wb')) #pickle.dump(fallback_dirs, open('fallback_dirs.p', 'wb'))

- # Calculate the number of fallback directory authorities present in the consensus and insert it into the database - fallback_dirs_running = 0 - fallback_dirs_notrunning = 0 - for relay_fp in consensuses.values()[0].routers: - if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags: - fallback_dirs_running += 1 - elif relay_fp in fallback_dirs: - fallback_dirs_notrunning += 1 - - insertValues = [unix_time(consensuses.values()[0].valid_after)] - insertValues.append(fallback_dirs_running) - insertValues.append(fallback_dirs_notrunning) - insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning)

dbc = sqlite3.connect(os.path.join('data', 'historical.db'))

- dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));") - dbc.commit() + # Calculate the number of fallback directory authorities present in the consensus and insert it into the database + if not CONFIG['ignore_fallback_authorities']: + fallback_dirs_running = 0 + fallback_dirs_notrunning = 0 + for relay_fp in consensuses.values()[0].routers: + if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags: + fallback_dirs_running += 1 + elif relay_fp in fallback_dirs: + fallback_dirs_notrunning += 1 + + insertValues = [unix_time(consensuses.values()[0].valid_after)] + insertValues.append(fallback_dirs_running) + insertValues.append(fallback_dirs_notrunning) + insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning) + + dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));") + dbc.commit()

- dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues) - dbc.commit() + dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues) + dbc.commit()

- # Write out the updated csv file for the graphs - fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160") - f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w') - f.write("date") - f.write(",fallback_dirs_running") - f.write(",fallback_dirs_notrunning") - f.write(",fallback_dirs_missing") - f.write("\n") - for r in fallback_dir_data.fetchall(): - for v in r: - f.write(("0" if v == None else str(v)) + ",") + # Write out the updated csv file for the graphs + fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160") + f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w') + f.write("date") + f.write(",fallback_dirs_running") + f.write(",fallback_dirs_notrunning") + f.write(",fallback_dirs_missing") f.write("\n") - f.close() + for r in fallback_dir_data.fetchall(): + for v in r: + f.write(("0" if v == None else str(v)) + ",") + f.write("\n") + f.close()

# Calculate the number of known and measured relays for each dirauth and insert it into the database data = {} @@ -130,10 +136,8 @@ def main(): createColumns = "" insertColumns = "date" insertQuestions = "" - import pdb - pdb.set_trace() - for dirauth_nickname in directory_authorities(): - dirauth_nickname = dirauth_nickname.lower() + + for dirauth_nickname in get_dirauths(): if vote_data_columns and dirauth_nickname not in vote_data_columns: dbc.execute("ALTER TABLE vote_data ADD COLUMN " + dirauth_nickname + "_known integer") dbc.execute("ALTER TABLE vote_data ADD COLUMN " + dirauth_nickname + "_running integer") @@ -155,12 +159,15 @@ def main(): dbc.commit()

# Write out the updated csv file for the graphs + vote_data_columns = [] + vote_data_schema = dbc.execute("PRAGMA table_info(vote_data)") + for c in vote_data_schema: + vote_data_columns.append(c[1]) + vote_data = dbc.execute("SELECT * from vote_data ORDER BY date DESC LIMIT 2160") f = open(os.path.join(os.path.dirname(__file__), 'out', 'vote-stats.csv'), 'w') - f.write("date") - for d in get_dirauths_in_tables(): - s = "," + d + "_known," + d + "_running," + d + "_bwauth" - f.write(s) + for c in vote_data_columns: + f.write(c + ",") f.write("\n") for r in vote_data.fetchall(): for v in r: @@ -168,11 +175,6 @@ def main(): f.write("\n") f.close()

- # great for debugging - #import pickle - #pickle.dump(consensuses, open('consensus.p', 'wb')) - #pickle.dump(votes, open('votes.p', 'wb')) - # produces the website w = WebsiteWriter() w.set_config(CONFIG) @@ -220,65 +222,6 @@ def main(): os.remove(os.path.join(os.path.dirname(__file__), 'out', f))

-def get_consensuses(): - """ - Provides a mapping of directory authority nicknames to their present consensus. - - :returns: tuple of the form ({authority => consensus}, issues, runtimes) - """ - - return _get_documents('consensus', '/tor/status-vote/current/consensus') - - -def get_votes(): - """ - Provides a mapping of directory authority nicknames to their present vote. - - :returns: tuple of the form ({authority => vote}, issues, runtimes) - """ - - return _get_documents('vote', '/tor/status-vote/current/authority') - - -def _get_documents(label, resource): - documents, issues, runtimes = {}, [], {} - - for authority in directory_authorities().values(): - if authority.v3ident is None: - continue # not a voting authority - - query = downloader.query( - resource, - endpoints = [(authority.address, authority.dir_port)], - default_params = False, - ) - - try: - start_time = time.time() - documents[authority.nickname] = query.run()[0] - runtimes[authority.nickname] = time.time() - start_time - except Exception, exc: - if label == 'vote': - # try to download the vote via the other authorities - - v3ident = directory_authorities()[authority.nickname].v3ident - - query = downloader.query( - '/tor/status-vote/current/%s' % v3ident, - default_params = False, - ) - - query.run(True) - - if not query.error: - documents[authority.nickname] = list(query)[0] - continue - - issues.append(('AUTHORITY_UNAVAILABLE', label, authority, query.download_url, exc)) - - return documents, issues, runtimes - - if __name__ == '__main__': try: main()