commit 668ed06e239bb85843f0813e2b763c7c7783d7a0 Author: Tom Ritter tom@ritter.vg Date: Fri Jul 1 12:41:04 2016 -0500
Generate a graphs page --- graphs.py | 416 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ write_website.py | 18 ++- 2 files changed, 431 insertions(+), 3 deletions(-)
diff --git a/graphs.py b/graphs.py new file mode 100755 index 0000000..c45c58e --- /dev/null +++ b/graphs.py @@ -0,0 +1,416 @@ +#!/usr/bin/env python +# See LICENSE for licensing information + +""" +Produces an HTML file for easily viewing voting and consensus differences +Ported from Java version Doctor +""" + +import os +import time +import operator +import datetime +import stem.descriptor.remote +from base64 import b64decode + +class GraphWriter: + consensus = None + votes = None + known_authorities = [] + consensus_expirey = datetime.timedelta(hours=3) + directory_key_warning_time = datetime.timedelta(days=14) + known_params = [] + def write_website(self, filename): + self.site = open(filename, 'w') + self._write_page_header() + self._write_valid_after_time() + self._write_number_of_relays_voted_about() + self._write_bandwidth_scanner_status() + self._write_bandwidth_scanner_graphs() + self._write_page_footer() + self.site.close() + + def set_consensuses(self, c): + self.consensuses = c + self.consensus = max(c.itervalues(), key=operator.attrgetter('valid_after')) + self.known_authorities = set([r.nickname for r in self.consensus.routers.values() if 'Authority' in r.flags and r.nickname != "Tonga"]) + self.known_authorities.update([r.nickname for r in self.consensus.directory_authorities]) + self.known_authorities.update([r for r in stem.descriptor.remote.get_authorities().keys() if r != "Tonga"]) + def set_votes(self, v): + self.votes = v + def set_consensus_expirey(self, timedelta): + self.consensus_expirey = timedelta + def set_directory_key_warning_time(self, timedelta): + self.directory_key_warning_time = timedelta + def set_config(self, config): + self.known_params = config['known_params'] + self.bandwidth_authorities = config['bandwidth_authorities'] + def get_consensus_time(self): + return self.consensus.valid_after + + #----------------------------------------------------------------------------------------- + def _write_page_header(self): + """ + Write the HTML page header including the metrics website navigation. + """ + self.site.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 " + + "Transitional//EN\">\n" + + "<html>\n" + + " <head>\n" + + " <title>Consensus health</title>\n" + + " <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1">\n" + + " <link href="stylesheet-ltr.css" type="text/css" rel="stylesheet">\n" + + " <link href="favicon.ico" type="image/x-icon" rel="shortcut icon">\n" + + " <script src="https://d3js.org/d3.v4.0.0-alpha.4.min.js%5C%22%3E</script>\n" + + " <script src="https://d3js.org/d3-dsv.v0.3.min.js%5C%22%3E</script>\n" + + " <script src="https://cdnjs.cloudflare.com/ajax/libs/d3-legend/1.10.0/d3-legend.min.js%5C%...</script>\n" + + " </head>\n" + + " <body>\n" + + " <style>\n" + + " svg {\n" + + " font: 10px sans-serif;\n" + + " }\n" + + " .axis path,\n" + + " .axis line {\n" + + " fill: none;\n" + + " stroke: #000;\n" + + " shape-rendering: crispEdges;\n" + + " }\n" + + " .graph-title {\n" + + " font-size: 16px;\n" + + " text-decoration: underline;\n" + + " }\n" + + " .faravahar_bwauth {\n" + + " fill: none;\n" + + " stroke: steelblue;\n" + + " background-color: steelblue;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " .gabelmoo_bwauth {\n" + + " fill: none;\n" + + " stroke: orange;\n" + + " background-color: orange;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " .moria1_bwauth {\n" + + " fill: none;\n" + + " stroke: yellow;\n" + + " background-color: yellow;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " .maatuska_bwauth {\n" + + " fill: none;\n" + + " stroke: green;\n" + + " background-color: green;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " .longclaw_bwauth {\n" + + " fill: none;\n" + + " stroke: red;\n" + + " background-color: red;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " </style>\n" + + " <div class="center">\n" + + " <div class="main-column">\n" + + " <h2>Consensus Health</h2>\n" + + " <br>\n" + + " <p>This page shows statistics about the current " + + "consensus and votes to facilitate debugging of the " + + "directory consensus process.") + self.site.write("</p>\n") + + #----------------------------------------------------------------------------------------- + def _write_valid_after_time(self): + """ + Write the valid-after time of the downloaded consensus. + """ + self.site.write("<br>\n\n\n" + + " <!-- ================================================================= -->" + + "<a name="validafter">\n" \ + + "<h3><a href="#validafter" class="anchor">" \ + + "Valid-after time</a></h3>\n" \ + + "<br>\n" \ + + "<p>Consensus was published ") + + if self.consensus.valid_after + self.consensus_expirey < datetime.datetime.now(): + self.site.write('<span class="oiv">' + + self.consensus.valid_after.isoformat().replace("T", " ") + + '</span>') + else: + self.site.write(self.consensus.valid_after.isoformat().replace("T", " ")) + + self.site.write(". <i>Note that it takes up to 15 minutes to learn " + + "about new consensus and votes and process them.</i></p>\n") + + #----------------------------------------------------------------------------------------- + def _write_number_of_relays_voted_about(self): + """ + Write the number of relays voted about. + """ + self.site.write("<br>\n\n\n" + + " <!-- ================================================================= -->" + + "<a name="numberofrelays">\n" + + "<h3><a href="#numberofrelays" class="anchor">" + + "Number of relays voted about</a></h3>\n" + + "<br>\n" + + "<table border="0" cellpadding="4" cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="320">\n" + + " <col width="320">\n" + + " </colgroup>\n") + if not self.votes: + self.site.write(" <tr><td>(No votes.)</td><td></td><td></td></tr>\n") + else: + for dirauth_nickname in self.known_authorities: + if dirauth_nickname in self.votes: + vote = self.votes[dirauth_nickname] + runningRelays = 0 + for r in vote.routers.values(): + if u'Running' in r.flags: + runningRelays += 1 + self.site.write(" <tr>\n" + + " <td>" + dirauth_nickname + "</td>\n" + + " <td>" + str(len(vote.routers)) + " total</td>\n" + + " <td>" + str(runningRelays) + " Running</td>\n" + + " </tr>\n") + else: + self.site.write(" <tr>\n" + + " <td>" + dirauth_nickname + "</td>\n" + + " <td colspan="2"><span class="oiv">Vote Not Present<span></td>\n" + + " </tr>\n") + runningRelays = 0 + for r in self.consensus.routers.values(): + if u'Running' in r.flags: + runningRelays += 1 + self.site.write(" <tr>\n" + + " <td class="ic">consensus</td>\n" + + " <td/>\n" + + " <td class="ic">" + str(runningRelays) + " Running</td>\n" + + " </tr>\n" + + "</table>\n") + + #----------------------------------------------------------------------------------------- + def _write_bandwidth_scanner_status(self): + """ + Write the status of bandwidth scanners and results being contained in votes. + """ + self.site.write("<br>\n\n\n" + + " <!-- ================================================================= -->" + + "<a name="bwauthstatus">\n" + + "<h3><a href="#bwauthstatus" class="anchor">" + + "Bandwidth scanner status</a></h3>\n" + + "<br>\n" + + "<table border="0" cellpadding="4" cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n") + if not self.votes: + self.site.write(" <tr><td>(No votes.)</td><td></td></tr>\n") + else: + for dirauth_nickname in self.votes: + vote = self.votes[dirauth_nickname] + + bandwidthWeights = 0 + for r in vote.routers.values(): + if r.measured >= 0L: + bandwidthWeights += 1 + + if bandwidthWeights > 0: + self.site.write(" <tr>\n" + + " <td>" + dirauth_nickname + "</td>\n" + + " <td>" + str(bandwidthWeights) + + " Measured values in w lines</td>\n" + + " </tr>\n") + for dirauth_nickname in self.bandwidth_authorities: + if dirauth_nickname not in self.votes: + self.site.write(" <tr>\n" + + " <td>" + dirauth_nickname + "</td>\n" + + " <td class="oiv">Missing vote</td>\n" + + " </tr>\n") + + self.site.write("</table>\n") + + #----------------------------------------------------------------------------------------- + def _write_bandwidth_scanner_graphs(self): + """ + Write the graphs of the bandwidth scanners + """ + self.site.write("<br>\n\n\n" + + " <!-- ================================================================= -->" + + "<a name="bwauthgraphs">\n" + + "<h3><a href="#bwauthstatus" class="anchor">" + + "Bandwidth scanner graphs</a></h3>\n" + + "<br>\n" + + "<table border="0" cellpadding="4" cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n" + + " <tr>\n" + + " <td>\n" + + " <div id="graphspot" style="text-align:center">\n" + + " <span class="moria1_bwauth" style="margin-left:5px"> </span> Moria\n" + + " <span class="faravahar_bwauth" style="margin-left:5px"> </span> Faravahar\n" + + " <span class="gabelmoo_bwauth" style="margin-left:5px"> </span> Gabelmoo\n" + + " <span class="maatuska_bwauth" style="margin-left:5px"> </span> Maatuska\n" + + " <span class="longclaw_bwauth" style="margin-left:5px"> </span> Longclaw\n" + + " </div>\n" + + " </td>\n" + + " </tr>\n" + + "</table>\n") + + s = """<script> + var BWAUTH_LOGICAL_MIN = 125 + var BWAUTHS = ["faravahar_bwauth","gabelmoo_bwauth","moria1_bwauth","maatuska_bwauth","longclaw_bwauth"]; + var WIDTH = 800, + HEIGHT = 500, + MARGIN = {top: 40, right: 40, bottom: 40, left: 40}; + + var GRAPHS_TO_GENERATE = [ + { title: "BWAuth Measured Relays, Past 30 Days", data_slice: 720 }, + { title: "BWAuth Measured Relays, Past 90 Days", data_slice: 1000 }, + { title: "BWAuth Measured Relays, Past Year", data_slice: 8760 }, + { title: "BWAuth Measured Relays, Past 2 Years", data_slice: 17520 }, + ]; + + fetch("https://ritter.vg/misc/stuff/bwauth_data.txt%22).then(function(response) { + return response.text(); + }).then(function(text) { + return d3_dsv.csvParse(text); + }).then(function(data) { + + for(g in GRAPHS_TO_GENERATE) + { + graph = GRAPHS_TO_GENERATE[g]; + + if(data.length-graph.data_slice > 0) + data_subset = data.slice(data.length-graph.data_slice); + else + data_subset = data + + min = 10000; + max = 0; + for(d in data_subset) + { + for(b in BWAUTHS) + { + data_subset[d][BWAUTHS[b]] = Number(data_subset[d][BWAUTHS[b]]); + if(data_subset[d][BWAUTHS[b]] < min && data_subset[d][BWAUTHS[b]] > BWAUTH_LOGICAL_MIN) { + min = data_subset[d][BWAUTHS[b]]; + } + if(data_subset[d][BWAUTHS[b]] > max) { + max = data_subset[d][BWAUTHS[b]]; + } + } + } + console.log("Data Length: " + data_subset.length + " Y-Axis Min: " + min + " Max: " + max); + + var x = d3.scaleTime() + .domain([new Date(Number(data_subset[0].date)), new Date(Number(data_subset[data_subset.length-1].date))]) + .range([0, WIDTH]) + ; + + var y = d3.scaleLinear() + .domain([min, max]) + .range([HEIGHT, 0]); + + var lines = [] + for(bwauth in BWAUTHS) + { + this_bwauth = BWAUTHS[bwauth]; + lines.push({bwauth: this_bwauth, line: (function(tmp) { + return d3.line() + .defined(function(d) { return d[tmp] && d[tmp] > BWAUTH_LOGICAL_MIN; }) + .x(function(d) { return x(new Date(Number(d.date))); }) + .y(function(d) { return y(d[tmp]); }); + })(this_bwauth)}); + } + + var svg = d3.select("#graphspot").append("svg") + .datum(data_subset) + .attr("width", WIDTH + MARGIN.left + MARGIN.right) + .attr("height", HEIGHT + MARGIN.top + MARGIN.bottom) + .append("g") + .attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")"); + + svg.append("g") + .attr("class", "axis axis--x") + .attr("transform", "translate(0," + HEIGHT + ")") + .call(d3.axisBottom().scale(x)); + + svg.append("g") + .attr("class", "axis axis--y") + .call(d3.axisLeft().scale(y)); + + for(l in lines) + { + svg.append("path") + .attr("class", lines[l].bwauth) + .attr("d", lines[l].line); + } + + svg.append("text") + .attr("x", (WIDTH / 2)) + .attr("y", 0 - (MARGIN.top / 2)) + .attr("text-anchor", "middle") + .attr("class", "graph-title") + .text(graph.title); + } + }); + + </script>""" + self.site.write(s) + + #----------------------------------------------------------------------------------------- + def _write_page_footer(self): + """ + Write the footer of the HTML page containing the blurb that is on + every page of the metrics website. + """ + #XXX Write the git version and stem version the page was generated with + self.site.write("</div>\n" + + "</div>\n" + + "<div class="bottom" id="bottom">\n" + + "<p>"Tor" and the "Onion Logo" are <a " + + "href="https://www.torproject.org/docs/trademark-faq.html.en%5C%22%3E" + + "registered trademarks</a> of The Tor Project, Inc.</p>\n" + + "</div>\n" + + "</body>\n" + + "</html>") + +if __name__ == '__main__': + """ + I found that the most effective way to test this independently was to pickle the + downloaded conensuses in ./write_website.py like this: + + import pickle + pickle.dump(consensuses, open('consensus.p', 'wb')) + pickle.dump(votes, open('votes.p', 'wb')) + + Then I can run ./website.py and pdb.set_trace() where needed to debug + """ + import stem + import pickle + g = GraphWriter() + + c = pickle.load(open('consensus.p', 'rb')) + g.set_consensuses(c) + v = pickle.load(open('votes.p', 'rb')) + g.set_votes(v) + + import pdb + pdb.set_trace() + + CONFIG = stem.util.conf.config_dict('consensus', { + 'ignored_authorities': [], + 'bandwidth_authorities': [], + 'known_params': [], + }) + config = stem.util.conf.get_config("consensus") + config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg')) + g.set_config(CONFIG) + + g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html')) diff --git a/write_website.py b/write_website.py index d998be1..85cca00 100755 --- a/write_website.py +++ b/write_website.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2013, Damian Johnson and The Tor Project +# Copyright 2013, Damian Johnson, Tom Ritter, and The Tor Project # See LICENSE for licensing information
""" @@ -23,6 +23,7 @@ from stem import Flag from stem.util.lru_cache import lru_cache
from website import WebsiteWriter +from graphs import GraphWriter
DIRECTORY_AUTHORITIES = stem.descriptor.remote.get_authorities()
@@ -71,10 +72,21 @@ def main(): w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'consensus-health.html'), True) w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'index.html'), False)
- # delete giant data structures for subprocess forking by piling hacks on top of each other consensus_time = w.get_consensus_time() + del w + + # produces the website + g = GraphWriter() + g.set_consensuses(consensuses) + g.set_votes(votes) + g.set_config(CONFIG) + g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html')) + + del g + + # delete giant data structures for subprocess forking by piling hacks on top of each other import gc - del w, consensuses, votes + del consensuses, votes gc.collect() time.sleep(1) archived = os.path.join(os.path.dirname(__file__), 'out', \
tor-commits@lists.torproject.org
-
tom＠torproject.org