commit 2c26327fe813513c66bb53227ac79744c497b9c1 Author: Tom Ritter tom@ritter.vg Date: Sun Oct 16 17:20:20 2016 -0400

Add fallback directory graphs and statistics --- graphs.py | 223 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- website.py | 50 +++++++++++++ write_website.py | 37 +++++++++ 3 files changed, 298 insertions(+), 12 deletions(-)

diff --git a/graphs.py b/graphs.py index 176ec27..9293b1f 100755 --- a/graphs.py +++ b/graphs.py @@ -21,6 +21,8 @@ class GraphWriter(WebsiteWriter): self.site = open(filename, 'w') self._write_page_header() self._write_valid_after_time() + self._write_fallback_directory_status(False) + self._write_fallback_directory_status_graphs() self._write_number_of_relays_voted_about(False) self._write_number_of_relays_voted_about_graphs() self._write_bandwidth_scanner_status(False) @@ -69,30 +71,39 @@ class GraphWriter(WebsiteWriter): + " background-color: steelblue;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .gabelmoo {\n" + + " .gabelmoo, .orange {\n" + " fill: none;\n" + " stroke: orange;\n" + " background-color: orange;\n" + " stroke-width: 1.5px;\n" + " }\n" + + " .orange {\n" + + " fill: orange;" + + " }\n" + " .moria1 {\n" + " fill: none;\n" + " stroke: yellow;\n" + " background-color: yellow;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .maatuska {\n" + + " .maatuska, .green {\n" + " fill: none;\n" + " stroke: green;\n" + " background-color: green;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .longclaw {\n" + + " .green {\n" + + " fill: green;" + + " }\n" + + " .longclaw, .red {\n" + " fill: none;\n" + " stroke: red;\n" + " background-color: red;\n" + " stroke-width: 1.5px;\n" + " }\n" + + " .red {\n" + + " fill: red;" + + " }\n" + " .tor26 {\n" + " fill: none;\n" + " stroke: purple;\n" @@ -134,6 +145,46 @@ class GraphWriter(WebsiteWriter): self.site.write("</p>\n") #----------------------------------------------------------------------------------------- + def _write_fallback_directory_status_graphs_spot(self, divName): + self.site.write(" <tr>\n" + + " <td>\n" + + " <div id="" + str(divName) + "" class="graphbox">\n" + + " <span class="green" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Running\n" + + " <span class="orange" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Not Running\n" + + " <span class="red" style="margin-left:5px">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Missing From Consensus\n" + + " </div>\n" + + " </td>\n" + + " </tr>\n") + def _write_fallback_directory_status_graphs(self): + """ + Write the graphs of the fallback directory mirrors + """ + self.site.write("<br>\n\n\n" + + " <!-- ================================================================= -->" + + "<a name="fallbackdirgraphs">\n" + + "<h3><a href="#fallbackdirgraphs" class="anchor">" + + "Fallback Directory graphs</a></h3>\n" + + "<br>\n" + + "<table border="0" cellpadding="4" cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n" + + " <tr class="graphplaceholder">\n" + + " <td>\n" + + " <div style="text-align:center">\n" + + " Generating Graph... (requires SVG and Javascript support)\n" + + " </div>\n" + + " </td>\n" + + " </tr>\n") + #self._write_fallback_directory_status_graphs_spot("fallbackdirs_pie") + self._write_fallback_directory_status_graphs_spot("fallbackdirs_1") + self._write_fallback_directory_status_graphs_spot("fallbackdirs_2") + self._write_fallback_directory_status_graphs_spot("fallbackdirs_3") + self._write_fallback_directory_status_graphs_spot("fallbackdirs_4") + self.site.write("</table>\n") + + #----------------------------------------------------------------------------------------- def _write_number_of_relays_voted_about_graphs_spot(self, divName): self.site.write(" <tr>\n" + " <td>\n" @@ -297,6 +348,19 @@ class GraphWriter(WebsiteWriter): */ ];

+ var FALLBACK_GRAPHS_TO_GENERATE = [ + { title: "Fallback Directories Running, Past 7 Days", data_slice: 168, div: "fallbackdirs_1", + data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, + { title: "Fallback Directories Running, Past 14 Days", data_slice: 336, div: "fallbackdirs_2", + data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, + { title: "Fallback Directories Running, Past 30 Days", data_slice: 720, div: "fallbackdirs_3", + data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, + { title: "Fallback Directories Running, Past 90 Days", data_slice: 2160, div: "fallbackdirs_4", + data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, + ]; + + relays_done = false; + fallbackdirs_done = false; fetch("vote-stats.csv").then(function(response) { return response.text(); }).then(function(text) { @@ -339,7 +403,7 @@ class GraphWriter(WebsiteWriter): if(x < min && x > graph.min_ignore_limit) min = x; if(x > max && x < graph.max_ignore_limit) - max = x; + max = x; if(x > graph.min_ignore_limit && x < graph.max_ignore_limit) { total += x; count++; @@ -411,23 +475,156 @@ class GraphWriter(WebsiteWriter): }

svg.append("text") - .attr("x", (WIDTH / 2)) + .attr("x", (WIDTH / 2)) .attr("y", 0 - (MARGIN.top / 2)) - .attr("text-anchor", "middle") - .attr("class", "graph-title") + .attr("text-anchor", "middle") + .attr("class", "graph-title") .text(graph.title); - } + }

+ relays_done = true; + if(fallbackdirs_done) { var toShow = document.getElementsByClassName('graphbox'); for(i=0; i<toShow.length; i++) { - console.log(toShow[i]); toShow[i].style.display = 'block'; } var toHide = document.getElementsByClassName('graphplaceholder'); for(i=0; i<toHide.length; i++) { - console.log(toHide[i]); toHide[i].style.display = 'none'; } + } + + }); + + fetch("fallback-dir-stats.csv").then(function(response) { + return response.text(); + }).then(function(text) { + return d3.csvParse(text, function(d) { + for(i in d) { + if(i == "date") + d[i] = new Date(Number(d[i])); + else + d[i] = Number(d[i]); + } + return d; + }); + }).then(function(data) { + var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' }; + /*Pie Graph + data_subset = data.slice(0); + data_subset = [ + {'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']}, + {'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']}, + {'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']}, + ]; + var data_func = function(d) { return d.value; }; + var arcs = d3.pie() + .sort(null) + .value(data_func)(data_subset); + + var svg = d3.select('#fallbackdirs_pie') + .append('svg') + .attr('width', WIDTH) + .attr('height', HEIGHT) + .append('g') + .attr('transform', 'translate(' + (WIDTH / 2) + ',' + (HEIGHT / 2) + ')'); + + var arc = d3.arc() + .innerRadius(0) + .outerRadius(100); + + var path = svg.selectAll('path') + .data(arcs) + .enter() + .append('path') + .attr('d', arc) + .attr('class', function(d, i) { + return key_to_color(d.data.label); + });*/ + + //Line Graphs + for(g in FALLBACK_GRAPHS_TO_GENERATE) + { + graph = FALLBACK_GRAPHS_TO_GENERATE[g]; + + if(graph.data_slice+1 > data.length) { + data_subset = data.slice(0); + console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items..."); + } + else + data_subset = data.slice(0, graph.data_slice); + data_subset.reverse(); + + max = 0 + for(d in data_subset) { + x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing']; + if(x > max) + max = x; + } + + var x = d3.scaleTime() + .domain([data_subset[0].date, data_subset[data_subset.length-1].date]) + .range([0, WIDTH]); + + var y = d3.scaleLinear() + .domain([0, max]) + .range([HEIGHT, 0]); + + var stack = d3.stack() + .keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"]) + .order(d3.stackOrderNone) + .offset(d3.stackOffsetNone); + + var area = d3.area() + .x(function(d, i) { return x(d.data.date); }) + .y0(function(d) { return y(d[0]); }) + .y1(function(d) { return y(d[1]); }); + + var svg = d3.select("#" + graph.div).append("svg") + .attr("width", WIDTH + MARGIN.left + MARGIN.right) + .attr("height", HEIGHT + MARGIN.top + MARGIN.bottom) + .append("g") + .attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")"); + + var layer = svg.selectAll(".layer") + .data(stack(data_subset)) + .enter().append("g") + //.attr("class", "layer"); + + layer.append("path") + //.attr("class", "area") + .attr("class", function(d) { return key_to_color(d.key); }) + .attr("d", area); + + svg.append("g") + .attr("class", "axis axis--x") + .attr("transform", "translate(0," + HEIGHT + ")") + .call(d3.axisBottom().scale(x)); + + svg.append("g") + .attr("class", "axis axis--y") + .call(d3.axisLeft().scale(y)); + + svg.append("text") + .attr("x", (WIDTH / 2)) + .attr("y", 0 - (MARGIN.top / 2)) + .attr("text-anchor", "middle") + .attr("class", "graph-title") + .text(graph.title); + } + + + fallbackdirs_done = true; + if(relays_done) { + var toShow = document.getElementsByClassName('graphbox'); + for(i=0; i<toShow.length; i++) { + toShow[i].style.display = 'block'; + } + var toHide = document.getElementsByClassName('graphplaceholder'); + for(i=0; i<toHide.length; i++) { + toHide[i].style.display = 'none'; + } + } });

</script>""" @@ -452,6 +649,9 @@ if __name__ == '__main__': g.set_consensuses(c) v = pickle.load(open('votes.p', 'rb')) g.set_votes(v) + f = pickle.load(open('fallback_dirs.p', 'rb')) + g.set_fallback_dirs(f) +

CONFIG = stem.util.conf.config_dict('consensus', { 'ignored_authorities': [], @@ -461,5 +661,4 @@ if __name__ == '__main__': config = stem.util.conf.get_config("consensus") config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg')) g.set_config(CONFIG) - - g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html')) + g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html')) \ No newline at end of file diff --git a/website.py b/website.py index b0eb68c..a634c16 100755 --- a/website.py +++ b/website.py @@ -580,6 +580,56 @@ class WebsiteWriter: self.site.write("</table>\n")

#----------------------------------------------------------------------------------------- + def _write_fallback_directory_status(self, linkToGraph): + """ + Write the status of the fallback directory mirrors + """ + self.site.write("<br>\n\n\n" + + " <!-- ================================================================= -->" + + "<a name="fallbackdirstatus">\n" + + "<h3><a href="#fallbackdirstatus" class="anchor">" + + "Fallback Directory status</a></h3>\n") + if linkToGraph: + self.site.write("<p>\n" + + " You can also view <a href="graphs.html">historical Fallback Directory graphs</a>.\n" + + "</p>\n") + else: + self.site.write("<br />\n") + self.site.write("<table border="0" cellpadding="4" cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n") + if not self.consensus: + self.site.write(" <tr><td>(No consensus.)</td><td></td></tr>\n") + else: + fallback_dirs_running = 0 + fallback_dirs_notrunning = 0 + fallback_dirs_missing = 0 + + for relay_fp in self.consensus.routers: + if relay_fp in self.fallback_dirs and self.consensus.routers[relay_fp].flags and 'Running' in self.consensus.routers[relay_fp].flags: + fallback_dirs_running += 1 + elif relay_fp in self.fallback_dirs: + fallback_dirs_notrunning += 1 + fallback_dirs_missing = len(self.fallback_dirs) - fallback_dirs_notrunning - fallback_dirs_running + + self.site.write(" <tr>\n" + + " <td>Running</td>\n" + + " <td>" + str(fallback_dirs_running) + "</td>\n" + + " </tr>\n") + self.site.write(" <tr>\n" + + " <td>Not Running</td>\n" + + " <td>" + str(fallback_dirs_notrunning) + "</td>\n" + + " </tr>\n") + self.site.write(" <tr>\n" + + " <td>Missing</td>\n" + + " <td>" + str(fallback_dirs_missing) + "</td>\n" + + " </tr>\n") + + self.site.write("</table>\n") + + #----------------------------------------------------------------------------------------- def _write_authority_versions(self): """ Write directory authority versions. diff --git a/write_website.py b/write_website.py index 8fca41e..2cf5de3 100755 --- a/write_website.py +++ b/write_website.py @@ -107,6 +107,42 @@ def main(): #import pickle #pickle.dump(fallback_dirs, open('fallback_dirs.p', 'wb'))

+ # Calculate the number of fallback directory authorities present in the consensus and insert it into the database + fallback_dirs_running = 0 + fallback_dirs_notrunning = 0 + for relay_fp in consensuses.values()[0].routers: + if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags: + fallback_dirs_running += 1 + elif relay_fp in fallback_dirs: + fallback_dirs_notrunning += 1 + + insertValues = [unix_time(consensuses.values()[0].valid_after)] + insertValues.append(fallback_dirs_running) + insertValues.append(fallback_dirs_notrunning) + insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning) + + dbc = sqlite3.connect(os.path.join('data', 'historical.db')) + + dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));") + dbc.commit() + + dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues) + dbc.commit() + + # Write out the updated csv file for the graphs + fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160") + f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w') + f.write("date") + f.write(",fallback_dirs_running") + f.write(",fallback_dirs_notrunning") + f.write(",fallback_dirs_missing") + f.write("\n") + for r in fallback_dir_data.fetchall(): + for v in r: + f.write(("0" if v == None else str(v)) + ",") + f.write("\n") + f.close() + # Calculate the number of known and measured relays for each dirauth and insert it into the database databaseDirAuths = "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ") data = {} @@ -172,6 +208,7 @@ def main(): g = GraphWriter() g.set_consensuses(consensuses) g.set_votes(votes) + g.set_fallback_dirs(fallback_dirs) g.set_config(CONFIG) g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html'))