[tor-commits] [depictor/master] Add fallback directory graphs and statistics

tom at torproject.org tom at torproject.org
Sun Oct 16 21:57:20 UTC 2016


commit 2c26327fe813513c66bb53227ac79744c497b9c1
Author: Tom Ritter <tom at ritter.vg>
Date:   Sun Oct 16 17:20:20 2016 -0400

    Add fallback directory graphs and statistics
---
 graphs.py        | 223 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
 website.py       |  50 +++++++++++++
 write_website.py |  37 +++++++++
 3 files changed, 298 insertions(+), 12 deletions(-)

diff --git a/graphs.py b/graphs.py
index 176ec27..9293b1f 100755
--- a/graphs.py
+++ b/graphs.py
@@ -21,6 +21,8 @@ class GraphWriter(WebsiteWriter):
 		self.site = open(filename, 'w')
 		self._write_page_header()
 		self._write_valid_after_time()
+		self._write_fallback_directory_status(False)
+		self._write_fallback_directory_status_graphs()
 		self._write_number_of_relays_voted_about(False)
 		self._write_number_of_relays_voted_about_graphs()
 		self._write_bandwidth_scanner_status(False)
@@ -69,30 +71,39 @@ class GraphWriter(WebsiteWriter):
 			+ "      background-color: steelblue;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .gabelmoo {\n"
+			+ "    .gabelmoo, .orange {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: orange;\n"
 			+ "      background-color: orange;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
+			+ "    .orange {\n"
+			+ "      fill: orange;"
+			+ "    }\n"
 			+ "    .moria1 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: yellow;\n"
 			+ "      background-color: yellow;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .maatuska {\n"
+			+ "    .maatuska, .green {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: green;\n"
 			+ "      background-color: green;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .longclaw {\n"
+			+ "    .green {\n"
+			+ "      fill: green;"
+			+ "    }\n"
+			+ "    .longclaw, .red {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: red;\n"
 			+ "      background-color: red;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
+			+ "    .red {\n"
+			+ "      fill: red;"
+			+ "    }\n"
 			+ "    .tor26 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: purple;\n"
@@ -134,6 +145,46 @@ class GraphWriter(WebsiteWriter):
 		self.site.write("</p>\n")
 		
 	#-----------------------------------------------------------------------------------------
+	def _write_fallback_directory_status_graphs_spot(self, divName):
+		self.site.write("  <tr>\n"
+		+ "    <td>\n"
+		+ "      <div id=\"" + str(divName) + "\" class=\"graphbox\">\n"
+        + "         <span class=\"green\" style=\"margin-left:5px\">     </span> Running\n"
+        + "         <span class=\"orange\" style=\"margin-left:5px\">     </span> Not Running\n"
+        + "         <span class=\"red\" style=\"margin-left:5px\">     </span> Missing From Consensus\n"
+		+ "      </div>\n"
+		+ "    </td>\n"
+		+ "  </tr>\n")
+	def _write_fallback_directory_status_graphs(self):
+		"""
+		Write the graphs of the fallback directory mirrors
+		"""
+		self.site.write("<br>\n\n\n"
+		+ " <!-- ================================================================= -->"
+		+ "<a name=\"fallbackdirgraphs\">\n"
+		+ "<h3><a href=\"#fallbackdirgraphs\" class=\"anchor\">"
+		+ "Fallback Directory graphs</a></h3>\n"
+		+ "<br>\n"
+		+ "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
+		+ "  <colgroup>\n"
+		+ "    <col width=\"160\">\n"
+		+ "    <col width=\"640\">\n"
+		+ "  </colgroup>\n"
+		+ "  <tr class=\"graphplaceholder\">\n"
+		+ "    <td>\n"
+		+ "      <div style=\"text-align:center\">\n"
+		+ "        Generating Graph... (requires SVG and Javascript support)\n"
+		+ "      </div>\n"
+		+ "    </td>\n"
+		+ "  </tr>\n")
+		#self._write_fallback_directory_status_graphs_spot("fallbackdirs_pie")
+		self._write_fallback_directory_status_graphs_spot("fallbackdirs_1")
+		self._write_fallback_directory_status_graphs_spot("fallbackdirs_2")
+		self._write_fallback_directory_status_graphs_spot("fallbackdirs_3")
+		self._write_fallback_directory_status_graphs_spot("fallbackdirs_4")
+		self.site.write("</table>\n")
+
+	#-----------------------------------------------------------------------------------------
 	def _write_number_of_relays_voted_about_graphs_spot(self, divName):
 		self.site.write("  <tr>\n"
 		+ "    <td>\n"
@@ -297,6 +348,19 @@ class GraphWriter(WebsiteWriter):
 */
 		];
 
+	    var FALLBACK_GRAPHS_TO_GENERATE = [
+	      { title: "Fallback Directories Running, Past 7 Days", data_slice: 168, div: "fallbackdirs_1", 
+	        data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX },
+	      { title: "Fallback Directories Running, Past 14 Days", data_slice: 336, div: "fallbackdirs_2", 
+	        data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX },
+	      { title: "Fallback Directories Running, Past 30 Days", data_slice: 720, div: "fallbackdirs_3", 
+	        data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX },
+	      { title: "Fallback Directories Running, Past 90 Days", data_slice: 2160, div: "fallbackdirs_4", 
+	        data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX },
+	    ];
+
+	    relays_done = false;
+	    fallbackdirs_done = false;
 		fetch("vote-stats.csv").then(function(response) {
 			return response.text();
 		}).then(function(text) {
@@ -339,7 +403,7 @@ class GraphWriter(WebsiteWriter):
 					if(x < min && x > graph.min_ignore_limit)
 						min = x;
 					if(x > max && x < graph.max_ignore_limit)
-						max = x;	
+						max = x;
 					if(x > graph.min_ignore_limit && x < graph.max_ignore_limit) {
 						total += x;
 						count++;
@@ -411,23 +475,156 @@ class GraphWriter(WebsiteWriter):
 			}
 
 			svg.append("text")
-			        .attr("x", (WIDTH / 2))             
+			        .attr("x", (WIDTH / 2))
 			        .attr("y", 0 - (MARGIN.top / 2))
-			        .attr("text-anchor", "middle")  
-			        .attr("class", "graph-title") 
+			        .attr("text-anchor", "middle")
+			        .attr("class", "graph-title")
 			        .text(graph.title);
-			}
+		}
 
+		relays_done = true;
+		if(fallbackdirs_done) {
 			var toShow = document.getElementsByClassName('graphbox');
 			for(i=0; i<toShow.length; i++) {
-				console.log(toShow[i]);
 				toShow[i].style.display = 'block';
 			}
 			var toHide = document.getElementsByClassName('graphplaceholder');
 			for(i=0; i<toHide.length; i++) {
-				console.log(toHide[i]);
 				toHide[i].style.display = 'none';
 			}
+		}
+
+		});
+
+		fetch("fallback-dir-stats.csv").then(function(response) {
+			return response.text();
+		}).then(function(text) {
+			return d3.csvParse(text, function(d) {
+				for(i in d) {
+					if(i == "date")
+						d[i] = new Date(Number(d[i]));
+					else
+						d[i] = Number(d[i]);
+				}
+				return d;
+			});
+		}).then(function(data) {
+			var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' };
+			/*Pie Graph
+			data_subset = data.slice(0);
+			data_subset = [
+				{'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']},
+				{'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']},
+				{'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']},
+			];
+			var data_func = function(d) { return d.value; };
+			var arcs = d3.pie()
+				.sort(null)
+				.value(data_func)(data_subset);
+
+			var svg = d3.select('#fallbackdirs_pie')
+				.append('svg')
+				.attr('width', WIDTH)
+				.attr('height', HEIGHT)
+				.append('g')
+				.attr('transform', 'translate(' + (WIDTH / 2) +	',' + (HEIGHT / 2) + ')');
+
+			var arc = d3.arc()
+				.innerRadius(0)
+				.outerRadius(100);
+
+			var path = svg.selectAll('path')
+				.data(arcs)
+				.enter()
+				.append('path')
+				.attr('d', arc)
+				.attr('class', function(d, i) {
+					return key_to_color(d.data.label);
+				});*/
+
+			//Line Graphs
+			for(g in FALLBACK_GRAPHS_TO_GENERATE)
+			{
+				graph = FALLBACK_GRAPHS_TO_GENERATE[g];
+
+				if(graph.data_slice+1 > data.length) {
+					data_subset = data.slice(0);
+					console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items...");
+				}
+				else
+					data_subset = data.slice(0, graph.data_slice);
+				data_subset.reverse();
+			
+				max = 0
+				for(d in data_subset) {
+					x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing'];
+					if(x > max)
+						max = x;
+				}
+
+				var x = d3.scaleTime()
+					.domain([data_subset[0].date, data_subset[data_subset.length-1].date])
+					.range([0, WIDTH]);
+
+				var y = d3.scaleLinear()
+					.domain([0, max])
+					.range([HEIGHT, 0]);
+
+				var stack = d3.stack()
+					.keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"])
+					.order(d3.stackOrderNone)
+					.offset(d3.stackOffsetNone);
+
+				var area = d3.area()
+					.x(function(d, i) { return x(d.data.date); })
+					.y0(function(d) { return y(d[0]); })
+					.y1(function(d) { return y(d[1]); });
+
+				var svg = d3.select("#" + graph.div).append("svg")
+					.attr("width", WIDTH + MARGIN.left + MARGIN.right)
+					.attr("height", HEIGHT + MARGIN.top + MARGIN.bottom)
+					.append("g")
+					.attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")");
+
+				var layer = svg.selectAll(".layer")
+					.data(stack(data_subset))
+					.enter().append("g")
+					//.attr("class", "layer");
+
+				layer.append("path")
+					//.attr("class", "area")
+					.attr("class", function(d) { return key_to_color(d.key); })
+					.attr("d", area);
+
+				svg.append("g")
+					.attr("class", "axis axis--x")
+					.attr("transform", "translate(0," + HEIGHT + ")")
+					.call(d3.axisBottom().scale(x));
+
+				svg.append("g")
+					.attr("class", "axis axis--y")
+					.call(d3.axisLeft().scale(y));
+
+				svg.append("text")
+					.attr("x", (WIDTH / 2))
+					.attr("y", 0 - (MARGIN.top / 2))
+					.attr("text-anchor", "middle")
+					.attr("class", "graph-title")
+					.text(graph.title);
+			}
+
+			
+			fallbackdirs_done = true;
+			if(relays_done) {
+				var toShow = document.getElementsByClassName('graphbox');
+				for(i=0; i<toShow.length; i++) {
+					toShow[i].style.display = 'block';
+				}
+				var toHide = document.getElementsByClassName('graphplaceholder');
+				for(i=0; i<toHide.length; i++) {
+					toHide[i].style.display = 'none';
+				}
+			}
 		});
 
 		</script>"""
@@ -452,6 +649,9 @@ if __name__ == '__main__':
 	g.set_consensuses(c)
 	v = pickle.load(open('votes.p', 'rb'))
 	g.set_votes(v)
+	f = pickle.load(open('fallback_dirs.p', 'rb'))
+	g.set_fallback_dirs(f)
+
 
 	CONFIG = stem.util.conf.config_dict('consensus', {
                                     'ignored_authorities': [],
@@ -461,5 +661,4 @@ if __name__ == '__main__':
 	config = stem.util.conf.get_config("consensus")
 	config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg'))
 	g.set_config(CONFIG)
-
-	g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html'))
+	g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html'))
\ No newline at end of file
diff --git a/website.py b/website.py
index b0eb68c..a634c16 100755
--- a/website.py
+++ b/website.py
@@ -580,6 +580,56 @@ class WebsiteWriter:
 		self.site.write("</table>\n")
 
 	#-----------------------------------------------------------------------------------------
+	def _write_fallback_directory_status(self, linkToGraph):
+		"""
+		Write the status of the fallback directory mirrors
+		"""
+		self.site.write("<br>\n\n\n"
+		+ " <!-- ================================================================= -->"
+		+ "<a name=\"fallbackdirstatus\">\n"
+		+ "<h3><a href=\"#fallbackdirstatus\" class=\"anchor\">"
+		+ "Fallback Directory status</a></h3>\n")
+		if linkToGraph:
+			self.site.write("<p>\n"
+			+ "  You can also view <a href=\"graphs.html\">historical Fallback Directory graphs</a>.\n"
+			+ "</p>\n")
+		else:
+			self.site.write("<br />\n")
+		self.site.write("<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
+		+ "  <colgroup>\n"
+		+ "    <col width=\"160\">\n"
+		+ "    <col width=\"640\">\n"
+		+ "  </colgroup>\n")
+		if not self.consensus:
+			self.site.write("  <tr><td>(No consensus.)</td><td></td></tr>\n")
+		else:
+			fallback_dirs_running = 0
+			fallback_dirs_notrunning = 0
+			fallback_dirs_missing = 0
+
+			for relay_fp in self.consensus.routers:
+				if relay_fp in self.fallback_dirs and self.consensus.routers[relay_fp].flags and 'Running' in self.consensus.routers[relay_fp].flags:
+					fallback_dirs_running += 1
+				elif relay_fp in self.fallback_dirs:
+					fallback_dirs_notrunning += 1
+			fallback_dirs_missing = len(self.fallback_dirs) - fallback_dirs_notrunning - fallback_dirs_running
+				
+			self.site.write("  <tr>\n"
+			+ "    <td>Running</td>\n"
+			+ "    <td>" + str(fallback_dirs_running) + "</td>\n"
+			+ "  </tr>\n")
+			self.site.write("  <tr>\n"
+			+ "    <td>Not Running</td>\n"
+			+ "    <td>" + str(fallback_dirs_notrunning) + "</td>\n"
+			+ "  </tr>\n")
+			self.site.write("  <tr>\n"
+			+ "    <td>Missing</td>\n"
+			+ "    <td>" + str(fallback_dirs_missing) + "</td>\n"
+			+ "  </tr>\n")
+
+		self.site.write("</table>\n")
+
+	#-----------------------------------------------------------------------------------------
 	def _write_authority_versions(self):
 		"""
 		Write directory authority versions.
diff --git a/write_website.py b/write_website.py
index 8fca41e..2cf5de3 100755
--- a/write_website.py
+++ b/write_website.py
@@ -107,6 +107,42 @@ def main():
 	#import pickle
 	#pickle.dump(fallback_dirs, open('fallback_dirs.p', 'wb'))
 
+	# Calculate the number of fallback directory authorities present in the consensus and insert it into the database
+	fallback_dirs_running = 0
+	fallback_dirs_notrunning = 0
+	for relay_fp in consensuses.values()[0].routers:
+		if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags:
+			fallback_dirs_running += 1
+		elif relay_fp in fallback_dirs:
+			fallback_dirs_notrunning += 1
+				
+	insertValues = [unix_time(consensuses.values()[0].valid_after)]
+	insertValues.append(fallback_dirs_running)
+	insertValues.append(fallback_dirs_notrunning)
+	insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning)
+
+	dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
+
+	dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));")
+	dbc.commit()
+
+	dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues)
+	dbc.commit()
+
+	# Write out the updated csv file for the graphs
+	fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160")
+	f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w')
+	f.write("date")
+	f.write(",fallback_dirs_running")
+	f.write(",fallback_dirs_notrunning")
+	f.write(",fallback_dirs_missing")
+	f.write("\n")
+	for r in fallback_dir_data.fetchall():
+		for v in r:
+			f.write(("0" if v == None else str(v)) + ",")
+		f.write("\n")
+	f.close()
+
 	# Calculate the number of known and measured relays for each dirauth and insert it into the database
 	databaseDirAuths = "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ")
 	data = {}
@@ -172,6 +208,7 @@ def main():
 	g = GraphWriter()
 	g.set_consensuses(consensuses)
 	g.set_votes(votes)
+	g.set_fallback_dirs(fallback_dirs)
 	g.set_config(CONFIG)
 	g.write_website(os.path.join(os.path.dirname(__file__), 'out', 'graphs.html'))
 





More information about the tor-commits mailing list