commit 2c26327fe813513c66bb53227ac79744c497b9c1
Author: Tom Ritter <tom(a)ritter.vg>
Date: Sun Oct 16 17:20:20 2016 -0400
Add fallback directory graphs and statistics
---
graphs.py | 223 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
website.py | 50 +++++++++++++
write_website.py | 37 +++++++++
3 files changed, 298 insertions(+), 12 deletions(-)
diff --git a/graphs.py b/graphs.py
index 176ec27..9293b1f 100755
--- a/graphs.py
+++ b/graphs.py
@@ -21,6 +21,8 @@ class GraphWriter(WebsiteWriter):
self.site = open(filename, 'w')
self._write_page_header()
self._write_valid_after_time()
+ self._write_fallback_directory_status(False)
+ self._write_fallback_directory_status_graphs()
self._write_number_of_relays_voted_about(False)
self._write_number_of_relays_voted_about_graphs()
self._write_bandwidth_scanner_status(False)
@@ -69,30 +71,39 @@ class GraphWriter(WebsiteWriter):
+ " background-color: steelblue;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .gabelmoo {\n"
+ + " .gabelmoo, .orange {\n"
+ " fill: none;\n"
+ " stroke: orange;\n"
+ " background-color: orange;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
+ + " .orange {\n"
+ + " fill: orange;"
+ + " }\n"
+ " .moria1 {\n"
+ " fill: none;\n"
+ " stroke: yellow;\n"
+ " background-color: yellow;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .maatuska {\n"
+ + " .maatuska, .green {\n"
+ " fill: none;\n"
+ " stroke: green;\n"
+ " background-color: green;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
- + " .longclaw {\n"
+ + " .green {\n"
+ + " fill: green;"
+ + " }\n"
+ + " .longclaw, .red {\n"
+ " fill: none;\n"
+ " stroke: red;\n"
+ " background-color: red;\n"
+ " stroke-width: 1.5px;\n"
+ " }\n"
+ + " .red {\n"
+ + " fill: red;"
+ + " }\n"
+ " .tor26 {\n"
+ " fill: none;\n"
+ " stroke: purple;\n"
@@ -134,6 +145,46 @@ class GraphWriter(WebsiteWriter):
self.site.write("</p>\n")
#-----------------------------------------------------------------------------------------
+ def _write_fallback_directory_status_graphs_spot(self, divName):
+ self.site.write(" <tr>\n"
+ + " <td>\n"
+ + " <div id=\"" + str(divName) + "\" class=\"graphbox\">\n"
+ + " <span class=\"green\" style=\"margin-left:5px\"> </span> Running\n"
+ + " <span class=\"orange\" style=\"margin-left:5px\"> </span> Not Running\n"
+ + " <span class=\"red\" style=\"margin-left:5px\"> </span> Missing From Consensus\n"
+ + " </div>\n"
+ + " </td>\n"
+ + " </tr>\n")
+ def _write_fallback_directory_status_graphs(self):
+ """
+ Write the graphs of the fallback directory mirrors
+ """
+ self.site.write("<br>\n\n\n"
+ + " <!-- ================================================================= -->"
+ + "<a name=\"fallbackdirgraphs\">\n"
+ + "<h3><a href=\"#fallbackdirgraphs\" class=\"anchor\">"
+ + "Fallback Directory graphs</a></h3>\n"
+ + "<br>\n"
+ + "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n"
+ + " <tr class=\"graphplaceholder\">\n"
+ + " <td>\n"
+ + " <div style=\"text-align:center\">\n"
+ + " Generating Graph... (requires SVG and Javascript support)\n"
+ + " </div>\n"
+ + " </td>\n"
+ + " </tr>\n")
+ #self._write_fallback_directory_status_graphs_spot("fallbackdirs_pie")
+ self._write_fallback_directory_status_graphs_spot("fallbackdirs_1")
+ self._write_fallback_directory_status_graphs_spot("fallbackdirs_2")
+ self._write_fallback_directory_status_graphs_spot("fallbackdirs_3")
+ self._write_fallback_directory_status_graphs_spot("fallbackdirs_4")
+ self.site.write("</table>\n")
+
+ #-----------------------------------------------------------------------------------------
def _write_number_of_relays_voted_about_graphs_spot(self, divName):
self.site.write(" <tr>\n"
+ " <td>\n"
@@ -297,6 +348,19 @@ class GraphWriter(WebsiteWriter):
*/
];
+ var FALLBACK_GRAPHS_TO_GENERATE = [
+ { title: "Fallback Directories Running, Past 7 Days", data_slice: 168, div: "fallbackdirs_1",
+ data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX },
+ { title: "Fallback Directories Running, Past 14 Days", data_slice: 336, div: "fallbackdirs_2",
+ data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX },
+ { title: "Fallback Directories Running, Past 30 Days", data_slice: 720, div: "fallbackdirs_3",
+ data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX },
+ { title: "Fallback Directories Running, Past 90 Days", data_slice: 2160, div: "fallbackdirs_4",
+ data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX },
+ ];
+
+ relays_done = false;
+ fallbackdirs_done = false;
fetch("vote-stats.csv").then(function(response) {
return response.text();
}).then(function(text) {
@@ -339,7 +403,7 @@ class GraphWriter(WebsiteWriter):
if(x < min && x > graph.min_ignore_limit)
min = x;
if(x > max && x < graph.max_ignore_limit)
- max = x;
+ max = x;
if(x > graph.min_ignore_limit && x < graph.max_ignore_limit) {
total += x;
count++;
@@ -411,23 +475,156 @@ class GraphWriter(WebsiteWriter):
}
svg.append("text")
- .attr("x", (WIDTH / 2))
+ .attr("x", (WIDTH / 2))
.attr("y", 0 - (MARGIN.top / 2))
- .attr("text-anchor", "middle")
- .attr("class", "graph-title")
+ .attr("text-anchor", "middle")
+ .attr("class", "graph-title")
.text(graph.title);
- }
+ }
+ relays_done = true;
+ if(fallbackdirs_done) {
var toShow = document.getElementsByClassName('graphbox');
for(i=0; i<toShow.length; i++) {
- console.log(toShow[i]);
toShow[i].style.display = 'block';
}
var toHide = document.getElementsByClassName('graphplaceholder');
for(i=0; i<toHide.length; i++) {
- console.log(toHide[i]);
toHide[i].style.display = 'none';
}
+ }
+
+ });
+
+ fetch("fallback-dir-stats.csv").then(function(response) {
+ return response.text();
+ }).then(function(text) {
+ return d3.csvParse(text, function(d) {
+ for(i in d) {
+ if(i == "date")
+ d[i] = new Date(Number(d[i]));
+ else
+ d[i] = Number(d[i]);
+ }
+ return d;
+ });
+ }).then(function(data) {
+ var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' };
+ /*Pie Graph
+ data_subset = data.slice(0);
+ data_subset = [
+ {'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']},
+ {'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']},
+ {'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']},
+ ];
+ var data_func = function(d) { return d.value; };
+ var arcs = d3.pie()
+ .sort(null)
+ .value(data_func)(data_subset);
+
+ var svg = d3.select('#fallbackdirs_pie')
+ .append('svg')
+ .attr('width', WIDTH)
+ .attr('height', HEIGHT)
+ .append('g')
+ .attr('transform', 'translate(' + (WIDTH / 2) + ',' + (HEIGHT / 2) + ')');
+
+ var arc = d3.arc()
+ .innerRadius(0)
+ .outerRadius(100);
+
+ var path = svg.selectAll('path')
+ .data(arcs)
+ .enter()
+ .append('path')
+ .attr('d', arc)
+ .attr('class', function(d, i) {
+ return key_to_color(d.data.label);
+ });*/
+
+ //Line Graphs
+ for(g in FALLBACK_GRAPHS_TO_GENERATE)
+ {
+ graph = FALLBACK_GRAPHS_TO_GENERATE[g];
+
+ if(graph.data_slice+1 > data.length) {
+ data_subset = data.slice(0);
+ console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items...");
+ }
+ else
+ data_subset = data.slice(0, graph.data_slice);
+ data_subset.reverse();
+
+ max = 0
+ for(d in data_subset) {
+ x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing'];
+ if(x > max)
+ max = x;
+ }
+
+ var x = d3.scaleTime()
+ .domain([data_subset[0].date, data_subset[data_subset.length-1].date])
+ .range([0, WIDTH]);
+
+ var y = d3.scaleLinear()
+ .domain([0, max])
+ .range([HEIGHT, 0]);
+
+ var stack = d3.stack()
+ .keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"])
+ .order(d3.stackOrderNone)
+ .offset(d3.stackOffsetNone);
+
+ var area = d3.area()
+ .x(function(d, i) { return x(d.data.date); })
+ .y0(function(d) { return y(d[0]); })
+ .y1(function(d) { return y(d[1]); });
+
+ var svg = d3.select("#" + graph.div).append("svg")
+ .attr("width", WIDTH + MARGIN.left + MARGIN.right)
+ .attr("height", HEIGHT + MARGIN.top + MARGIN.bottom)
+ .append("g")
+ .attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")");
+
+ var layer = svg.selectAll(".layer")
+ .data(stack(data_subset))
+ .enter().append("g")
+ //.attr("class", "layer");
+
+ layer.append("path")
+ //.attr("class", "area")
+ .attr("class", function(d) { return key_to_color(d.key); })
+ .attr("d", area);
+
+ svg.append("g")
+ .attr("class", "axis axis--x")
+ .attr("transform", "translate(0," + HEIGHT + ")")
+ .call(d3.axisBottom().scale(x));
+
+ svg.append("g")
+ .attr("class", "axis axis--y")
+ .call(d3.axisLeft().scale(y));
+
+ svg.append("text")
+ .attr("x", (WIDTH / 2))
+ .attr("y", 0 - (MARGIN.top / 2))
+ .attr("text-anchor", "middle")
+ .attr("class", "graph-title")
+ .text(graph.title);
+ }
+
+
+ fallbackdirs_done = true;
+ if(relays_done) {
+ var toShow = document.getElementsByClassName('graphbox');
+ for(i=0; i<toShow.length; i++) {
+ toShow[i].style.display = 'block';
+ }
+ var toHide = document.getElementsByClassName('graphplaceholder');
+ for(i=0; i<toHide.length; i++) {
+ toHide[i].style.display = 'none';
+ }
+ }
});
</script>"""
@@ -452,6 +649,9 @@ if __name__ == '__main__':
g.set_consensuses(c)
v = pickle.load(open('votes.p', 'rb'))
g.set_votes(v)
+ f = pickle.load(open('fallback_dirs.p', 'rb'))
+ g.set_fallback_dirs(f)
+
CONFIG = stem.util.conf.config_dict('consensus', {
'ignored_authorities': [],
@@ -461,5 +661,4 @@ if __name__ == '__main__':
config = stem.util.conf.get_config("consensus")
config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg'))
g.set_config(CONFIG)
-
- g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html'))
+ g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html'))
\ No newline at end of file
diff --git a/website.py b/website.py
index b0eb68c..a634c16 100755
--- a/website.py
+++ b/website.py
@@ -580,6 +580,56 @@ class WebsiteWriter:
self.site.write("</table>\n")
#-----------------------------------------------------------------------------------------
+ def _write_fallback_directory_status(self, linkToGraph):
+ """
+ Write the status of the fallback directory mirrors
+ """
+ self.site.write("<br>\n\n\n"
+ + " <!-- ================================================================= -->"
+ + "<a name=\"fallbackdirstatus\">\n"
+ + "<h3><a href=\"#fallbackdirstatus\" class=\"anchor\">"
+ + "Fallback Directory status</a></h3>\n")
+ if linkToGraph:
+ self.site.write("<p>\n"
+ + " You can also view <a href=\"graphs.html\">historical Fallback Directory graphs</a>.\n"
+ + "</p>\n")
+ else:
+ self.site.write("<br />\n")
+ self.site.write("<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n")
+ if not self.consensus:
+ self.site.write(" <tr><td>(No consensus.)</td><td></td></tr>\n")
+ else:
+ fallback_dirs_running = 0
+ fallback_dirs_notrunning = 0
+ fallback_dirs_missing = 0
+
+ for relay_fp in self.consensus.routers:
+ if relay_fp in self.fallback_dirs and self.consensus.routers[relay_fp].flags and 'Running' in self.consensus.routers[relay_fp].flags:
+ fallback_dirs_running += 1
+ elif relay_fp in self.fallback_dirs:
+ fallback_dirs_notrunning += 1
+ fallback_dirs_missing = len(self.fallback_dirs) - fallback_dirs_notrunning - fallback_dirs_running
+
+ self.site.write(" <tr>\n"
+ + " <td>Running</td>\n"
+ + " <td>" + str(fallback_dirs_running) + "</td>\n"
+ + " </tr>\n")
+ self.site.write(" <tr>\n"
+ + " <td>Not Running</td>\n"
+ + " <td>" + str(fallback_dirs_notrunning) + "</td>\n"
+ + " </tr>\n")
+ self.site.write(" <tr>\n"
+ + " <td>Missing</td>\n"
+ + " <td>" + str(fallback_dirs_missing) + "</td>\n"
+ + " </tr>\n")
+
+ self.site.write("</table>\n")
+
+ #-----------------------------------------------------------------------------------------
def _write_authority_versions(self):
"""
Write directory authority versions.
diff --git a/write_website.py b/write_website.py
index 8fca41e..2cf5de3 100755
--- a/write_website.py
+++ b/write_website.py
@@ -107,6 +107,42 @@ def main():
#import pickle
#pickle.dump(fallback_dirs, open('fallback_dirs.p', 'wb'))
+ # Calculate the number of fallback directory authorities present in the consensus and insert it into the database
+ fallback_dirs_running = 0
+ fallback_dirs_notrunning = 0
+ for relay_fp in consensuses.values()[0].routers:
+ if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags:
+ fallback_dirs_running += 1
+ elif relay_fp in fallback_dirs:
+ fallback_dirs_notrunning += 1
+
+ insertValues = [unix_time(consensuses.values()[0].valid_after)]
+ insertValues.append(fallback_dirs_running)
+ insertValues.append(fallback_dirs_notrunning)
+ insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning)
+
+ dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
+
+ dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));")
+ dbc.commit()
+
+ dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues)
+ dbc.commit()
+
+ # Write out the updated csv file for the graphs
+ fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160")
+ f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w')
+ f.write("date")
+ f.write(",fallback_dirs_running")
+ f.write(",fallback_dirs_notrunning")
+ f.write(",fallback_dirs_missing")
+ f.write("\n")
+ for r in fallback_dir_data.fetchall():
+ for v in r:
+ f.write(("0" if v == None else str(v)) + ",")
+ f.write("\n")
+ f.close()
+
# Calculate the number of known and measured relays for each dirauth and insert it into the database
databaseDirAuths = "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ")
data = {}
@@ -172,6 +208,7 @@ def main():
g = GraphWriter()
g.set_consensuses(consensuses)
g.set_votes(votes)
+ g.set_fallback_dirs(fallback_dirs)
g.set_config(CONFIG)
g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html'))
