commit 0a39f7f7d40fe09172cee67db35b66e5ec804297 Author: Tom Ritter <tom@ritter.vg> Date: Tue Jul 5 21:59:03 2016 -0500 Add more graphs, abstract the graph code more to more easily add new ones, fix some CSV parsing issues --- graphs.py | 198 +++++++++++++++++++++++++++++++++++++++++++++++-------- write_website.py | 2 +- 2 files changed, 171 insertions(+), 29 deletions(-) diff --git a/graphs.py b/graphs.py index 3c17b0b..b44158a 100755 --- a/graphs.py +++ b/graphs.py @@ -22,6 +22,7 @@ class GraphWriter(WebsiteWriter): self._write_page_header() self._write_valid_after_time() self._write_number_of_relays_voted_about() + self._write_number_of_relays_voted_about_graphs() self._write_bandwidth_scanner_status(False) self._write_bandwidth_scanner_graphs() self._write_graph_javascript() @@ -60,6 +61,10 @@ class GraphWriter(WebsiteWriter): + " font-size: 16px;\n" + " text-decoration: underline;\n" + " }\n" + + " .graphbox {\n" + + " text-align: center;\n" + #+ " display: none;\n" + + " }\n" + " .faravahar {\n" + " fill: none;\n" + " stroke: steelblue;\n" @@ -90,6 +95,36 @@ class GraphWriter(WebsiteWriter): + " background-color: red;\n" + " stroke-width: 1.5px;\n" + " }\n" + + " .tor26 {\n" + + " fill: none;\n" + + " stroke: purple;\n" + + " background-color: purple;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " .urras {\n" + + " fill: none;\n" + + " stroke: black;\n" + + " background-color: black;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " .turtles {\n" + + " fill: none;\n" + + " stroke: #0000FF;\n" + + " background-color: #0000FF;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " .dizum {\n" + + " fill: none;\n" + + " stroke: limegreen;\n" + + " background-color: limegreen;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + + " .dannenberg {\n" + + " fill: none;\n" + + " stroke: pink;\n" + + " background-color: pink;\n" + + " stroke-width: 1.5px;\n" + + " }\n" + " </style>\n" + " <div class=\"center\">\n" + " <div class=\"main-column\">\n" @@ -101,19 +136,65 @@ class GraphWriter(WebsiteWriter): self.site.write("</p>\n") #----------------------------------------------------------------------------------------- - def _write_bandwidth_scanner_graphs_spot(self, divName): + def _write_number_of_relays_voted_about_graphs_spot(self, divName): self.site.write(" <tr>\n" + " <td>\n" - + " <div id=\"" + str(divName) + "\" style=\"text-align:center\">\n" + + " <div id=\"" + str(divName) + "\" class=\"graphbox\">\n" + " <span class=\"moria1\" style=\"margin-left:5px\">     </span> Moria\n" + " <span class=\"faravahar\" style=\"margin-left:5px\">     </span> Faravahar\n" + " <span class=\"gabelmoo\" style=\"margin-left:5px\">     </span> Gabelmoo\n" + " <span class=\"maatuska\" style=\"margin-left:5px\">     </span> Maatuska\n" + " <span class=\"longclaw\" style=\"margin-left:5px\">     </span> Longclaw\n" + + " <span class=\"tor26\" style=\"margin-left:5px\">     </span> tor26\n" + + " <span class=\"urras\" style=\"margin-left:5px\">     </span> urras\n" + + " <span class=\"dizum\" style=\"margin-left:5px\">     </span> dizum\n" + + " <span class=\"dannenberg\" style=\"margin-left:5px\">     </span> dannenberg\n" + + " <span class=\"turtles\" style=\"margin-left:5px\">     </span> turtles\n" + " </div>\n" + " </td>\n" + " </tr>\n") + def _write_number_of_relays_voted_about_graphs(self): + """ + Write the graphs of the number of relays voted about + """ + self.site.write("<br>\n\n\n" + + " <!-- ================================================================= -->" + + "<a name=\"votedaboutgraphs\">\n" + + "<h3><a href=\"#votedaboutgraphs\" class=\"anchor\">" + + "Number of relays voted about graphs</a></h3>\n" + + "<br>\n" + + "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n" + + " <colgroup>\n" + + " <col width=\"160\">\n" + + " <col width=\"640\">\n" + + " </colgroup>\n") + self._write_number_of_relays_voted_about_graphs_spot("voted_total_1") + self._write_number_of_relays_voted_about_graphs_spot("voted_total_2") + self._write_number_of_relays_voted_about_graphs_spot("voted_total_3") + self._write_number_of_relays_voted_about_graphs_spot("voted_total_4") + self._write_number_of_relays_voted_about_graphs_spot("voted_running_1") + self._write_number_of_relays_voted_about_graphs_spot("voted_running_2") + self._write_number_of_relays_voted_about_graphs_spot("voted_running_3") + self._write_number_of_relays_voted_about_graphs_spot("voted_running_4") + self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_1") + self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_2") + self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_3") + self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_4") + self.site.write("</table>\n") + #----------------------------------------------------------------------------------------- + def _write_bandwidth_scanner_graphs_spot(self, divName): + self.site.write(" <tr>\n" + + " <td>\n" + + " <div id=\"" + str(divName) + "\" class=\"graphbox\">\n" + + " <span class=\"moria1\" style=\"margin-left:5px\">     </span> Moria\n" + + " <span class=\"faravahar\" style=\"margin-left:5px\">     </span> Faravahar\n" + + " <span class=\"gabelmoo\" style=\"margin-left:5px\">     </span> Gabelmoo\n" + + " <span class=\"maatuska\" style=\"margin-left:5px\">     </span> Maatuska\n" + + " <span class=\"longclaw\" style=\"margin-left:5px\">     </span> Longclaw\n" + + " </div>\n" + + " </td>\n" + + " </tr>\n") def _write_bandwidth_scanner_graphs(self): """ Write the graphs of the bandwidth scanners @@ -121,7 +202,7 @@ class GraphWriter(WebsiteWriter): self.site.write("<br>\n\n\n" + " <!-- ================================================================= -->" + "<a name=\"bwauthgraphs\">\n" - + "<h3><a href=\"#bwauthstatus\" class=\"anchor\">" + + "<h3><a href=\"#bwauthgraphs\" class=\"anchor\">" + "Bandwidth scanner measured relays</a></h3>\n" + "<br>\n" + "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n" @@ -133,34 +214,87 @@ class GraphWriter(WebsiteWriter): self._write_bandwidth_scanner_graphs_spot("bwauth_measured_2") self._write_bandwidth_scanner_graphs_spot("bwauth_measured_3") self._write_bandwidth_scanner_graphs_spot("bwauth_measured_4") + #self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_1") + #self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_2") + #self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_3") + #self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_4") self.site.write("</table>\n") def _write_graph_javascript(self): s = """<script> - var BWAUTH_LOGICAL_MIN = 125 + var AUTH_LOGICAL_MIN = 125 var WIDTH = 800, HEIGHT = 500, MARGIN = {top: 40, right: 40, bottom: 40, left: 40}; var bwauths = ["faravahar","gabelmoo","moria1","maatuska","longclaw"]; var dirauths = """ + str(get_dirauths_in_tables()) + """; - var _getBandwidthDataValue = function(d, dirauth) { d[dirauth + "_bwauth"] = Number(d[dirauth + "_bwauth"]); return d[dirauth + "_bwauth"]; } + var _getBandwidthDataValue = function(d, dirauth) { return d[dirauth + "_bwauth"]; } + var _getRunningDataValue = function(d, dirauth) { return d[dirauth + "_running"]; } + var _getTotalDataValue = function(d, dirauth) { return d[dirauth + "_known"]; } + var _getNonRunningDataValue = function(d, dirauth) { return d[dirauth + "_known"] - d[dirauth + "_running"]; } + var _getRunningUnmeasuredDataValue = function(d, dirauth) { return d[dirauth + "_running"] - d[dirauth + "_bwauth"]; } var GRAPHS_TO_GENERATE = [ - { title: "BWAuth Measured Relays, Past 30 Days", data_slice: 720, div: "bwauth_measured_1", - data_func: _getBandwidthDataValue, authorities: bwauths }, - { title: "BWAuth Measured Relays, Past 90 Days", data_slice: 1000, div: "bwauth_measured_2", - data_func: _getBandwidthDataValue, authorities: bwauths }, - { title: "BWAuth Measured Relays, Past Year", data_slice: 8760, div: "bwauth_measured_3", - data_func: _getBandwidthDataValue, authorities: bwauths }, - { title: "BWAuth Measured Relays, Past 2 Years", data_slice: 17520, div: "bwauth_measured_4", - data_func: _getBandwidthDataValue, authorities: bwauths }, + { title: "Voted About Relays (Running), Past 7 Days", data_slice: 168, div: "voted_running_1", + data_func: _getRunningDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "Voted About Relays (Running), Past 14 Days", data_slice: 336, div: "voted_running_2", + data_func: _getRunningDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "Voted About Relays (Running), Past 30 Days", data_slice: 720, div: "voted_running_3", + data_func: _getRunningDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "Voted About Relays (Running), Past 90 Days", data_slice: 2160, div: "voted_running_4", + data_func: _getRunningDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + + { title: "Voted About Relays (Total), Past 7 Days", data_slice: 168, div: "voted_total_1", + data_func: _getTotalDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "Voted About Relays (Total), Past 14 Days", data_slice: 336, div: "voted_total_2", + data_func: _getTotalDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "Voted About Relays (Total), Past 30 Days", data_slice: 720, div: "voted_total_3", + data_func: _getTotalDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "Voted About Relays (Total), Past 90 Days", data_slice: 2160, div: "voted_total_4", + data_func: _getTotalDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + + { title: "Voted About Relays (Not Running), Past 7 Days", data_slice: 168, div: "voted_notrunning_1", + data_func: _getNonRunningDataValue, authorities: dirauths, ignore_limit:0 }, + { title: "Voted About Relays (Not Running), Past 14 Days", data_slice: 336, div: "voted_notrunning_2", + data_func: _getNonRunningDataValue, authorities: dirauths, ignore_limit:0 }, + { title: "Voted About Relays (Not Running), Past 30 Days", data_slice: 720, div: "voted_notrunning_3", + data_func: _getNonRunningDataValue, authorities: dirauths, ignore_limit:0 }, + { title: "Voted About Relays (Not Running), Past 90 Days", data_slice: 2160, div: "voted_notrunning_4", + data_func: _getNonRunningDataValue, authorities: dirauths, ignore_limit:0 }, + + { title: "BWAuth Measured Relays, Past 7 Days", data_slice: 168, div: "bwauth_measured_1", + data_func: _getBandwidthDataValue, authorities: bwauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "BWAuth Measured Relays, Past 14 Days", data_slice: 336, div: "bwauth_measured_2", + data_func: _getBandwidthDataValue, authorities: bwauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "BWAuth Measured Relays, Past 30 Days", data_slice: 720, div: "bwauth_measured_3", + data_func: _getBandwidthDataValue, authorities: bwauths, ignore_limit:AUTH_LOGICAL_MIN }, + { title: "BWAuth Measured Relays, Past 90 Days", data_slice: 2160, div: "bwauth_measured_4", + data_func: _getBandwidthDataValue, authorities: bwauths, ignore_limit:AUTH_LOGICAL_MIN }, +/* These graphs are very misleading and not helpful + { title: "BWAuth Running Unmeasured Relays, Past 30 Days", data_slice: 720, div: "bwauth_running_unmeasured_1", + data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, ignore_limit:-1000 }, + { title: "BWAuth Running Unmeasured Relays, Past 90 Days", data_slice: 2160, div: "bwauth_running_unmeasured_2", + data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, ignore_limit:-1000 }, + { title: "BWAuth Running Unmeasured Relays, Past Year", data_slice: 8760, div: "bwauth_running_unmeasured_3", + data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, ignore_limit:-1000 }, + { title: "BWAuth Running Unmeasured Relays, Past 2 Years", data_slice: 17520, div: "bwauth_running_unmeasured_4", + data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, ignore_limit:-1000 }, +*/ ]; - fetch("https://ritter.vg/misc/stuff/bwauth_data.txt").then(function(response) { + fetch("https://ritter.vg/misc/stuff/vote-stats.csv").then(function(response) { return response.text(); }).then(function(text) { - return d3_dsv.csvParse(text); + return d3_dsv.csvParse(text, function(d) { + for(i in d) { + if(i == "date") + d[i] = new Date(Number(d[i])); + else + d[i] = Number(d[i]); + } + return d; + }); }).then(function(data) { // For each of the configured graphs @@ -168,10 +302,13 @@ class GraphWriter(WebsiteWriter): { graph = GRAPHS_TO_GENERATE[g]; - if(data.length-graph.data_slice > 0) - data_subset = data.slice(data.length-graph.data_slice); + if(graph.data_slice+1 > data.length) { + data_subset = data.slice(1); + console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items..."); + } else - data_subset = data + data_subset = data.slice(1, graph.data_slice+1); + data_subset.reverse(); // Calculate the Graph Boundaries ----------------------------------------- min = 10000; @@ -183,13 +320,16 @@ class GraphWriter(WebsiteWriter): for(a in graph.authorities) { var x = graph.data_func(data_subset[d], graph.authorities[a]); - if(x < min && x > BWAUTH_LOGICAL_MIN) + if(isNaN(x)) + console.log("Error, NAN:", data_subset[d], graph.authorities[a], x); + if(x < min && x > graph.ignore_limit) min = x; if(x > max) max = x; - - total += x; - count++; + if(x > graph.ignore_limit) { + total += x; + count++; + } } } avg = total / count; @@ -199,21 +339,23 @@ class GraphWriter(WebsiteWriter): for(a in graph.authorities) { var x = graph.data_func(data_subset[d], graph.authorities[a]); - sumvariance += (x - avg) * (x - avg); + if(x > graph.ignore_limit) { + sumvariance += (x - avg) * (x - avg); + } } } variance = sumvariance / count; stddev = Math.sqrt(variance); - console.log("Data Length: " + data_subset.length + " Y-Axis Min: " + min + " Max: " + max + " Avg: " + avg + " Var: " + variance + " StdDev: " + stddev); + console.log("("+graph.title+") Data Length: " + data_subset.length + " Y-Axis Min: " + min + " Max: " + max + " Avg: " + avg + " Var: " + variance + " StdDev: " + stddev); // Create the Graph ----------------------------------------- var x = d3.scaleTime() - .domain([new Date(Number(data_subset[0].date)), new Date(Number(data_subset[data_subset.length-1].date))]) + .domain([data_subset[0].date, data_subset[data_subset.length-1].date]) .range([0, WIDTH]) ; var y = d3.scaleLinear() - .domain([avg-(stddev), avg+(stddev)]) + .domain([avg-(5*stddev), avg+(5*stddev)]) .range([HEIGHT, 0]); var lines = [] @@ -222,8 +364,8 @@ class GraphWriter(WebsiteWriter): this_auth = graph.authorities[auth]; lines.push({auth: this_auth, line: (function(dirAuthClosure) { return d3.line() - .defined(function(d) { return graph.data_func(d, dirAuthClosure) && graph.data_func(d, dirAuthClosure) > BWAUTH_LOGICAL_MIN; }) - .x(function(d) { return x(new Date(Number(d.date))); }) + .defined(function(d) { return d && graph.data_func(d, dirAuthClosure) && graph.data_func(d, dirAuthClosure) > graph.ignore_limit; }) + .x(function(d) { return x(d.date); }) .y(function(d) { return y(graph.data_func(d, dirAuthClosure)); }); })(this_auth)}); } diff --git a/write_website.py b/write_website.py index 247b95c..25781b0 100755 --- a/write_website.py +++ b/write_website.py @@ -99,7 +99,7 @@ def main(): f = open(os.path.join(os.path.dirname(__file__), 'out', 'vote-stats.csv'), 'w') f.write("date") for d in get_dirauths_in_tables(): - s = "," + d + "_known, " + d + "_running, " + d + "_bwauth" + s = "," + d + "_known," + d + "_running," + d + "_bwauth" f.write(s) f.write("\n") for r in vote_data.fetchall():