September 2016 - tor-commits - lists.torproject.org

[depictor/master] Make the graph's y-axis the standard deviation instead of the min/max.
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit 90396074f5666f56176e1eac3a7f4260ccdba039 Author: Tom Ritter <tom(a)ritter.vg> Date: Tue Jul 5 12:50:26 2016 -0500 Make the graph's y-axis the standard deviation instead of the min/max. --- graphs.py | 38 +++++++++++++++++++++++++++----------- parseOldConsensuses.py | 5 ++++- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/graphs.py b/graphs.py index c45c58e..8b55c15 100755 --- a/graphs.py +++ b/graphs.py @@ -282,6 +282,7 @@ class GraphWriter: return d3_dsv.csvParse(text); }).then(function(data) { + // For each of the configured graphs for(g in GRAPHS_TO_GENERATE) { graph = GRAPHS_TO_GENERATE[g]; @@ -291,30 +292,48 @@ class GraphWriter: else data_subset = data + // Calculate the Graph Boundaries ----------------------------------------- min = 10000; max = 0; + total = 0; + count = 0; for(d in data_subset) { for(b in BWAUTHS) { data_subset[d][BWAUTHS[b]] = Number(data_subset[d][BWAUTHS[b]]); - if(data_subset[d][BWAUTHS[b]] < min && data_subset[d][BWAUTHS[b]] > BWAUTH_LOGICAL_MIN) { - min = data_subset[d][BWAUTHS[b]]; - } - if(data_subset[d][BWAUTHS[b]] > max) { - max = data_subset[d][BWAUTHS[b]]; - } + var x = data_subset[d][BWAUTHS[b]]; + if(x < min && x > BWAUTH_LOGICAL_MIN) + min = x; + if(x > max) + max = x; + + total += x; + count++; } } - console.log("Data Length: " + data_subset.length + " Y-Axis Min: " + min + " Max: " + max); + avg = total / count; + sumvariance = 0; + for(d in data_subset) + { + for(b in BWAUTHS) + { + var x = data_subset[d][BWAUTHS[b]]; + sumvariance += (x - avg) * (x - avg); + } + } + variance = sumvariance / count; + stddev = Math.sqrt(variance); + console.log("Data Length: " + data_subset.length + " Y-Axis Min: " + min + " Max: " + max + " Avg: " + avg + " Var: " + variance + " StdDev: " + stddev); + // Create the Graph ----------------------------------------- var x = d3.scaleTime() .domain([new Date(Number(data_subset[0].date)), new Date(Number(data_subset[data_subset.length-1].date))]) .range([0, WIDTH]) ; var y = d3.scaleLinear() - .domain([min, max]) + .domain([avg-(stddev), avg+(stddev)]) .range([HEIGHT, 0]); var lines = [] @@ -401,9 +420,6 @@ if __name__ == '__main__': v = pickle.load(open('votes.p', 'rb')) g.set_votes(v) - import pdb - pdb.set_trace() - CONFIG = stem.util.conf.config_dict('consensus', { 'ignored_authorities': [], 'bandwidth_authorities': [], diff --git a/parseOldConsensuses.py b/parseOldConsensuses.py index e45fd27..834c386 100755 --- a/parseOldConsensuses.py +++ b/parseOldConsensuses.py @@ -20,6 +20,9 @@ import stem.util.enum from stem import Flag from stem.util.lru_cache import lru_cache +def get_dirauths_in_tables(): + return "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ") + def get_dirauth_from_filename(filename): key = filename.split('-') @@ -64,7 +67,7 @@ def get_time_from_filename(filename): return voteTime def main(dir): - dirAuths = "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ") + dirAuths = get_dirauths_in_tables() dbc = sqlite3.connect(os.path.join('data', 'historical.db')) dirauth_columns = ""

1 0

[depictor/master] Add a logical maximum for the graphs to avoid spikes in subtraction graphs, repoint the graphs at the local vote-stats.csv data
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit 8feb5ade803a1b1c980b11e9372f02310a5a766a Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Jul 6 13:35:00 2016 -0400 Add a logical maximum for the graphs to avoid spikes in subtraction graphs, repoint the graphs at the local vote-stats.csv data --- data/historical.db | Bin 2191360 -> 2191360 bytes graphs.py | 63 +++++++++++++++++++++++++++++------------------------ 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/data/historical.db b/data/historical.db index 9146c8b..fd4b938 100644 Binary files a/data/historical.db and b/data/historical.db differ diff --git a/graphs.py b/graphs.py index 4ceb6d3..e36ca45 100755 --- a/graphs.py +++ b/graphs.py @@ -232,13 +232,17 @@ class GraphWriter(WebsiteWriter): def _write_graph_javascript(self): s = """<script> - var AUTH_LOGICAL_MIN = 125 + var AUTH_LOGICAL_MIN = 125, + AUTH_LOGICAL_MAX = 25000; var WIDTH = 800, HEIGHT = 500, - MARGIN = {top: 40, right: 40, bottom: 40, left: 40}; + MARGIN = {top: 40, right: 40, bottom: 40, left: 40}; var bwauths = ["faravahar","gabelmoo","moria1","maatuska","longclaw"]; var dirauths = """ + str(get_dirauths_in_tables()) + """; + dirauths.splice(dirauths.indexOf('urras'), 1); + dirauths.splice(dirauths.indexOf('turtles'), 1); + var _getBandwidthDataValue = function(d, dirauth) { return d[dirauth + "_bwauth"]; } var _getRunningDataValue = function(d, dirauth) { return d[dirauth + "_running"]; } var _getTotalDataValue = function(d, dirauth) { return d[dirauth + "_known"]; } @@ -247,53 +251,53 @@ class GraphWriter(WebsiteWriter): var GRAPHS_TO_GENERATE = [ { title: "Voted About Relays (Running), Past 7 Days", data_slice: 168, div: "voted_running_1", - data_func: _getRunningDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "Voted About Relays (Running), Past 14 Days", data_slice: 336, div: "voted_running_2", - data_func: _getRunningDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "Voted About Relays (Running), Past 30 Days", data_slice: 720, div: "voted_running_3", - data_func: _getRunningDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "Voted About Relays (Running), Past 90 Days", data_slice: 2160, div: "voted_running_4", - data_func: _getRunningDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getRunningDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "Voted About Relays (Total), Past 7 Days", data_slice: 168, div: "voted_total_1", - data_func: _getTotalDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getTotalDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "Voted About Relays (Total), Past 14 Days", data_slice: 336, div: "voted_total_2", - data_func: _getTotalDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getTotalDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "Voted About Relays (Total), Past 30 Days", data_slice: 720, div: "voted_total_3", - data_func: _getTotalDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getTotalDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "Voted About Relays (Total), Past 90 Days", data_slice: 2160, div: "voted_total_4", - data_func: _getTotalDataValue, authorities: dirauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getTotalDataValue, authorities: dirauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "Voted About Relays (Not Running), Past 7 Days", data_slice: 168, div: "voted_notrunning_1", - data_func: _getNonRunningDataValue, authorities: dirauths, ignore_limit:0 }, + data_func: _getNonRunningDataValue, authorities: dirauths, min_ignore_limit:0, max_ignore_limit:4000 }, { title: "Voted About Relays (Not Running), Past 14 Days", data_slice: 336, div: "voted_notrunning_2", - data_func: _getNonRunningDataValue, authorities: dirauths, ignore_limit:0 }, + data_func: _getNonRunningDataValue, authorities: dirauths, min_ignore_limit:0, max_ignore_limit:4000 }, { title: "Voted About Relays (Not Running), Past 30 Days", data_slice: 720, div: "voted_notrunning_3", - data_func: _getNonRunningDataValue, authorities: dirauths, ignore_limit:0 }, + data_func: _getNonRunningDataValue, authorities: dirauths, min_ignore_limit:0, max_ignore_limit:4000 }, { title: "Voted About Relays (Not Running), Past 90 Days", data_slice: 2160, div: "voted_notrunning_4", - data_func: _getNonRunningDataValue, authorities: dirauths, ignore_limit:0 }, + data_func: _getNonRunningDataValue, authorities: dirauths, min_ignore_limit:0, max_ignore_limit:4000 }, { title: "BWAuth Measured Relays, Past 7 Days", data_slice: 168, div: "bwauth_measured_1", - data_func: _getBandwidthDataValue, authorities: bwauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getBandwidthDataValue, authorities: bwauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "BWAuth Measured Relays, Past 14 Days", data_slice: 336, div: "bwauth_measured_2", - data_func: _getBandwidthDataValue, authorities: bwauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getBandwidthDataValue, authorities: bwauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "BWAuth Measured Relays, Past 30 Days", data_slice: 720, div: "bwauth_measured_3", - data_func: _getBandwidthDataValue, authorities: bwauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getBandwidthDataValue, authorities: bwauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "BWAuth Measured Relays, Past 90 Days", data_slice: 2160, div: "bwauth_measured_4", - data_func: _getBandwidthDataValue, authorities: bwauths, ignore_limit:AUTH_LOGICAL_MIN }, + data_func: _getBandwidthDataValue, authorities: bwauths, min_ignore_limit:AUTH_LOGICAL_MIN, max_ignore_limit:AUTH_LOGICAL_MAX }, /* These graphs are very misleading and not helpful { title: "BWAuth Running Unmeasured Relays, Past 30 Days", data_slice: 720, div: "bwauth_running_unmeasured_1", - data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, ignore_limit:-1000 }, + data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, min_ignore_limit:-1000, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "BWAuth Running Unmeasured Relays, Past 90 Days", data_slice: 2160, div: "bwauth_running_unmeasured_2", - data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, ignore_limit:-1000 }, + data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, min_ignore_limit:-1000, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "BWAuth Running Unmeasured Relays, Past Year", data_slice: 8760, div: "bwauth_running_unmeasured_3", - data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, ignore_limit:-1000 }, + data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, min_ignore_limit:-1000, max_ignore_limit:AUTH_LOGICAL_MAX }, { title: "BWAuth Running Unmeasured Relays, Past 2 Years", data_slice: 17520, div: "bwauth_running_unmeasured_4", - data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, ignore_limit:-1000 }, + data_func: _getRunningUnmeasuredDataValue, authorities: bwauths, min_ignore_limit:-1000, max_ignore_limit:AUTH_LOGICAL_MAX }, */ ]; - fetch("https://ritter.vg/misc/stuff/vote-stats.csv").then(function(response) { + fetch("vote-stats.csv").then(function(response) { return response.text(); }).then(function(text) { return d3.csvParse(text, function(d) { @@ -332,11 +336,11 @@ class GraphWriter(WebsiteWriter): var x = graph.data_func(data_subset[d], graph.authorities[a]); if(isNaN(x)) console.log("Error, NAN:", data_subset[d], graph.authorities[a], x); - if(x < min && x > graph.ignore_limit) + if(x < min && x > graph.min_ignore_limit) min = x; - if(x > max) + if(x > max && x < graph.max_ignore_limit) max = x; - if(x > graph.ignore_limit) { + if(x > graph.min_ignore_limit && x < graph.max_ignore_limit) { total += x; count++; } @@ -349,7 +353,7 @@ class GraphWriter(WebsiteWriter): for(a in graph.authorities) { var x = graph.data_func(data_subset[d], graph.authorities[a]); - if(x > graph.ignore_limit) { + if(x > graph.min_ignore_limit && x < graph.max_ignore_limit) { sumvariance += (x - avg) * (x - avg); } } @@ -374,7 +378,10 @@ class GraphWriter(WebsiteWriter): this_auth = graph.authorities[auth]; lines.push({auth: this_auth, line: (function(dirAuthClosure) { return d3.line() - .defined(function(d) { return d && graph.data_func(d, dirAuthClosure) && graph.data_func(d, dirAuthClosure) > graph.ignore_limit; }) + .defined(function(d) { + return d && graph.data_func(d, dirAuthClosure) && + graph.data_func(d, dirAuthClosure) > graph.min_ignore_limit && + graph.data_func(d, dirAuthClosure) < graph.max_ignore_limit; }) .x(function(d) { return x(d.date); }) .y(function(d) { return y(graph.data_func(d, dirAuthClosure)); }); })(this_auth)});

1 0

[depictor/master] Handle turtles as a DirAuth and change the duplicate vote handling to not be an abort. Move the DB construction lower.
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit dc6f97b03d882b4d68265a24770fd99dc24bb637 Author: Tom Ritter <tom(a)ritter.vg> Date: Fri Jul 1 13:47:22 2016 -0400 Handle turtles as a DirAuth and change the duplicate vote handling to not be an abort. Move the DB construction lower. --- parseOldConsensuses.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/parseOldConsensuses.py b/parseOldConsensuses.py index 66e11eb..a616014 100755 --- a/parseOldConsensuses.py +++ b/parseOldConsensuses.py @@ -45,6 +45,8 @@ def get_dirauth_from_filename(filename): return "tor26" elif key == "0232AF901C31A04EE9848595AF9BB7620D4C5B2E" or key == "585769C78764D58426B8B52B6651A5A71137189A": return "dannenberg" + elif key == "27B6B5996C426270A5C95488AA5BCEB6BCC86956": + return "turtles" else: raise Exception("Unexpcected dirauth key: " + key + " " + filename) @@ -70,16 +72,17 @@ def main(dir): for d in dirAuths: dirauth_columns += d + "_running integer, " + d + "_bwauth integer, " dirauth_columns_questions += ", ?, ?" - dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " + dirauth_columns + "PRIMARY KEY(date ASC))") - dbc.commit() votes = {} for root, dirs, files in os.walk(dir): for f in files: - print os.path.join(root, f) + filepath = os.path.join(root, f) + print filepath if '"' in f: raise Exception("Potentially malicious filename") + elif "votes-" in f and ".tar" in f: + continue voteTime = get_time_from_filename(f) if voteTime not in votes: @@ -91,12 +94,14 @@ def main(dir): elif dirauth not in votes[voteTime]: votes[voteTime][dirauth] = {} else: - raise Exception("Found two votes for dirauth " + dirauth + " and time " + filename) + print "Found two votes for dirauth " + dirauth + " and time " + filepath votes[voteTime][dirauth]['present'] = 1 - votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + os.path.join(root, f) + '" | wc -l', shell=True)) - votes[voteTime][dirauth]['running'] = int(subprocess.check_output('egrep "^s " "' + os.path.join(root, f) + '" | grep " Running" | wc -l', shell=True)) + votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l', shell=True)) + votes[voteTime][dirauth]['running'] = int(subprocess.check_output('egrep "^s " "' + filepath + '" | grep " Running" | wc -l', shell=True)) + dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " + dirauth_columns + "PRIMARY KEY(date ASC))") + dbc.commit() for t in votes: print t

1 0

[depictor/master] Abstract the graph-making javascript to handle more graphs
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit 3631dc6204fae77cd7ce76914b40a3e5030d977b Author: Tom Ritter <tom(a)ritter.vg> Date: Tue Jul 5 17:00:15 2016 -0500 Abstract the graph-making javascript to handle more graphs --- graphs.py | 64 +++++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/graphs.py b/graphs.py index 8336c36..3c17b0b 100755 --- a/graphs.py +++ b/graphs.py @@ -14,6 +14,7 @@ import stem.descriptor.remote from base64 import b64decode from website import WebsiteWriter +from parseOldConsensuses import get_dirauths_in_tables class GraphWriter(WebsiteWriter): def write_website(self, filename): @@ -59,31 +60,31 @@ class GraphWriter(WebsiteWriter): + " font-size: 16px;\n" + " text-decoration: underline;\n" + " }\n" - + " .faravahar_bwauth {\n" + + " .faravahar {\n" + " fill: none;\n" + " stroke: steelblue;\n" + " background-color: steelblue;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .gabelmoo_bwauth {\n" + + " .gabelmoo {\n" + " fill: none;\n" + " stroke: orange;\n" + " background-color: orange;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .moria1_bwauth {\n" + + " .moria1 {\n" + " fill: none;\n" + " stroke: yellow;\n" + " background-color: yellow;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .maatuska_bwauth {\n" + + " .maatuska {\n" + " fill: none;\n" + " stroke: green;\n" + " background-color: green;\n" + " stroke-width: 1.5px;\n" + " }\n" - + " .longclaw_bwauth {\n" + + " .longclaw {\n" + " fill: none;\n" + " stroke: red;\n" + " background-color: red;\n" @@ -104,11 +105,11 @@ class GraphWriter(WebsiteWriter): self.site.write(" <tr>\n" + " <td>\n" + " <div id=\"" + str(divName) + "\" style=\"text-align:center\">\n" - + " <span class=\"moria1_bwauth\" style=\"margin-left:5px\">     </span> Moria\n" - + " <span class=\"faravahar_bwauth\" style=\"margin-left:5px\">     </span> Faravahar\n" - + " <span class=\"gabelmoo_bwauth\" style=\"margin-left:5px\">     </span> Gabelmoo\n" - + " <span class=\"maatuska_bwauth\" style=\"margin-left:5px\">     </span> Maatuska\n" - + " <span class=\"longclaw_bwauth\" style=\"margin-left:5px\">     </span> Longclaw\n" + + " <span class=\"moria1\" style=\"margin-left:5px\">     </span> Moria\n" + + " <span class=\"faravahar\" style=\"margin-left:5px\">     </span> Faravahar\n" + + " <span class=\"gabelmoo\" style=\"margin-left:5px\">     </span> Gabelmoo\n" + + " <span class=\"maatuska\" style=\"margin-left:5px\">     </span> Maatuska\n" + + " <span class=\"longclaw\" style=\"margin-left:5px\">     </span> Longclaw\n" + " </div>\n" + " </td>\n" + " </tr>\n") @@ -134,18 +135,26 @@ class GraphWriter(WebsiteWriter): self._write_bandwidth_scanner_graphs_spot("bwauth_measured_4") self.site.write("</table>\n") + def _write_graph_javascript(self): s = """<script> var BWAUTH_LOGICAL_MIN = 125 - var BWAUTHS = ["faravahar_bwauth","gabelmoo_bwauth","moria1_bwauth","maatuska_bwauth","longclaw_bwauth"]; var WIDTH = 800, HEIGHT = 500, MARGIN = {top: 40, right: 40, bottom: 40, left: 40}; + var bwauths = ["faravahar","gabelmoo","moria1","maatuska","longclaw"]; + var dirauths = """ + str(get_dirauths_in_tables()) + """; + var _getBandwidthDataValue = function(d, dirauth) { d[dirauth + "_bwauth"] = Number(d[dirauth + "_bwauth"]); return d[dirauth + "_bwauth"]; } + var GRAPHS_TO_GENERATE = [ - { title: "BWAuth Measured Relays, Past 30 Days", data_slice: 720 }, - { title: "BWAuth Measured Relays, Past 90 Days", data_slice: 1000 }, - { title: "BWAuth Measured Relays, Past Year", data_slice: 8760 }, - { title: "BWAuth Measured Relays, Past 2 Years", data_slice: 17520 }, + { title: "BWAuth Measured Relays, Past 30 Days", data_slice: 720, div: "bwauth_measured_1", + data_func: _getBandwidthDataValue, authorities: bwauths }, + { title: "BWAuth Measured Relays, Past 90 Days", data_slice: 1000, div: "bwauth_measured_2", + data_func: _getBandwidthDataValue, authorities: bwauths }, + { title: "BWAuth Measured Relays, Past Year", data_slice: 8760, div: "bwauth_measured_3", + data_func: _getBandwidthDataValue, authorities: bwauths }, + { title: "BWAuth Measured Relays, Past 2 Years", data_slice: 17520, div: "bwauth_measured_4", + data_func: _getBandwidthDataValue, authorities: bwauths }, ]; fetch("https://ritter.vg/misc/stuff/bwauth_data.txt").then(function(response) { @@ -171,10 +180,9 @@ class GraphWriter(WebsiteWriter): count = 0; for(d in data_subset) { - for(b in BWAUTHS) + for(a in graph.authorities) { - data_subset[d][BWAUTHS[b]] = Number(data_subset[d][BWAUTHS[b]]); - var x = data_subset[d][BWAUTHS[b]]; + var x = graph.data_func(data_subset[d], graph.authorities[a]); if(x < min && x > BWAUTH_LOGICAL_MIN) min = x; if(x > max) @@ -188,9 +196,9 @@ class GraphWriter(WebsiteWriter): sumvariance = 0; for(d in data_subset) { - for(b in BWAUTHS) + for(a in graph.authorities) { - var x = data_subset[d][BWAUTHS[b]]; + var x = graph.data_func(data_subset[d], graph.authorities[a]); sumvariance += (x - avg) * (x - avg); } } @@ -209,18 +217,18 @@ class GraphWriter(WebsiteWriter): .range([HEIGHT, 0]); var lines = [] - for(bwauth in BWAUTHS) + for(auth in graph.authorities) { - this_bwauth = BWAUTHS[bwauth]; - lines.push({bwauth: this_bwauth, line: (function(tmp) { + this_auth = graph.authorities[auth]; + lines.push({auth: this_auth, line: (function(dirAuthClosure) { return d3.line() - .defined(function(d) { return d[tmp] && d[tmp] > BWAUTH_LOGICAL_MIN; }) + .defined(function(d) { return graph.data_func(d, dirAuthClosure) && graph.data_func(d, dirAuthClosure) > BWAUTH_LOGICAL_MIN; }) .x(function(d) { return x(new Date(Number(d.date))); }) - .y(function(d) { return y(d[tmp]); }); - })(this_bwauth)}); + .y(function(d) { return y(graph.data_func(d, dirAuthClosure)); }); + })(this_auth)}); } - var svg = d3.select("#graphspot").append("svg") + var svg = d3.select("#" + graph.div).append("svg") .datum(data_subset) .attr("width", WIDTH + MARGIN.left + MARGIN.right) .attr("height", HEIGHT + MARGIN.top + MARGIN.bottom) @@ -239,7 +247,7 @@ class GraphWriter(WebsiteWriter): for(l in lines) { svg.append("path") - .attr("class", lines[l].bwauth) + .attr("class", lines[l].auth) .attr("d", lines[l].line); }

1 0

[depictor/master] Parse old consensus votes and put them into a sqlite db
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit 3ada6a96541e3c7e0f3cc43668c1860f543b7d93 Author: Tom Ritter <tom(a)ritter.vg> Date: Fri Jun 24 15:49:41 2016 -0500 Parse old consensus votes and put them into a sqlite db --- parseOldConsensuses.py | 129 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/parseOldConsensuses.py b/parseOldConsensuses.py new file mode 100755 index 0000000..66e11eb --- /dev/null +++ b/parseOldConsensuses.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# Copyright 2013, Damian Johnson, Tom Ritter, and The Tor Project +# See LICENSE for licensing information + + +import os +import sys +import time +import sqlite3 +import datetime +import operator +import traceback +import subprocess + +import stem.descriptor +import stem.descriptor.remote +import stem.util.conf +import stem.util.enum + +from stem import Flag +from stem.util.lru_cache import lru_cache + + +def get_dirauth_from_filename(filename): + key = filename.split('-') + if len(key) < 9: + raise Exception("Strange filename: " + filename) + + key = key[-2] + if key == "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97": + return "faravahar" + elif key == "ED03BB616EB2F60BEC80151114BB25CEF515B226": + return "gabelmoo" + elif key == "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58": + return "dizum" + elif key == "D586D18309DED4CD6D57C18FDB97EFA96D330566": + return "moria1" + elif key == "80550987E1D626E3EBA5E5E75A458DE0626D088C": + return "urras" + elif key == "49015F787433103580E3B66A1707A00E60F2D15B": + return "maatuska" + elif key == "23D15D965BC35114467363C165C4F724B64B4F66": + return "longclaw" + elif key == "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4": + return "tor26" + elif key == "0232AF901C31A04EE9848595AF9BB7620D4C5B2E" or key == "585769C78764D58426B8B52B6651A5A71137189A": + return "dannenberg" + else: + raise Exception("Unexpcected dirauth key: " + key + " " + filename) + +def unix_time(dt): + return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0 + +def get_time_from_filename(filename): + voteTime = filename.split('-') + if len(voteTime) < 9: + raise Exception("Strange filename: " + filename) + + v = [int(x) for x in filename.split('-')[0:6]] + voteTime = datetime.datetime(v[0], v[1], v[2], v[3], v[4], v[5]) + voteTime = unix_time(voteTime) + return voteTime + +def main(dir): + dirAuths = "faravahar, gabelmoo, dizum, moria1, urras, maatuska, longclaw, tor26, dannenberg, turtles".split(", ") + dbc = sqlite3.connect(os.path.join('data', 'historical.db')) + + dirauth_columns = "" + dirauth_columns_questions = "" + for d in dirAuths: + dirauth_columns += d + "_running integer, " + d + "_bwauth integer, " + dirauth_columns_questions += ", ?, ?" + dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " + dirauth_columns + "PRIMARY KEY(date ASC))") + dbc.commit() + + votes = {} + for root, dirs, files in os.walk(dir): + for f in files: + print os.path.join(root, f) + + if '"' in f: + raise Exception("Potentially malicious filename") + + voteTime = get_time_from_filename(f) + if voteTime not in votes: + votes[voteTime] = {} + + dirauth = get_dirauth_from_filename(f) + if dirauth not in dirAuths: + raise Exception("Found a dirauth I don't know about (probably spelling): " + dirauth) + elif dirauth not in votes[voteTime]: + votes[voteTime][dirauth] = {} + else: + raise Exception("Found two votes for dirauth " + dirauth + " and time " + filename) + + votes[voteTime][dirauth]['present'] = 1 + votes[voteTime][dirauth]['bwlines'] = int(subprocess.check_output('grep Measured= "' + os.path.join(root, f) + '" | wc -l', shell=True)) + votes[voteTime][dirauth]['running'] = int(subprocess.check_output('egrep "^s " "' + os.path.join(root, f) + '" | grep " Running" | wc -l', shell=True)) + + + for t in votes: + print t + print "\t", len(votes[t]) + for d in votes[t]: + print "\t", d, votes[t][d]['bwlines'], votes[t][d]['running'] + + insertValues = [t] + for d in dirAuths: + if d in votes[t]: + insertValues.append(votes[t][d]['running']) + insertValues.append(votes[t][d]['bwlines']) + else: + insertValues.append(None) + insertValues.append(None) + + dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?" + dirauth_columns_questions + ")", insertValues) + dbc.commit() + + +if __name__ == '__main__': + try: + if len(sys.argv) != 2: + print "Usage: ", sys.argv[0], "vote-directory" + else: + main(sys.argv[1]) + except: + msg = "%s failed with:\n\n%s" % (sys.argv[0], traceback.format_exc()) + print "Error: %s" % msg +

1 0

[depictor/master] Merge branch 'vote-tracking' of /home/tom/git/depictor into vote-tracking
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit 4b7624ea937ffc47e701d400455637e544516a45 Merge: dc6f97b 668ed06 Author: Tom Ritter <tom(a)ritter.vg> Date: Fri Jul 1 13:47:47 2016 -0400 Merge branch 'vote-tracking' of /home/tom/git/depictor into vote-tracking graphs.py | 416 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ write_website.py | 18 ++- 2 files changed, 431 insertions(+), 3 deletions(-)

1 0

[depictor/master] Add placeholders for graphs while they are generating, link from the main index page, and reduce the size of the vote data in the csv file
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit a2b0104c621e7172554cc3e78844d562a6470549 Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Jul 6 11:57:39 2016 -0500 Add placeholders for graphs while they are generating, link from the main index page, and reduce the size of the vote data in the csv file --- graphs.py | 35 +++++++++++++++++++++++++++++------ website.py | 15 ++++++++++----- write_website.py | 2 +- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/graphs.py b/graphs.py index ac07c97..4ceb6d3 100755 --- a/graphs.py +++ b/graphs.py @@ -21,7 +21,7 @@ class GraphWriter(WebsiteWriter): self.site = open(filename, 'w') self._write_page_header() self._write_valid_after_time() - self._write_number_of_relays_voted_about() + self._write_number_of_relays_voted_about(False) self._write_number_of_relays_voted_about_graphs() self._write_bandwidth_scanner_status(False) self._write_bandwidth_scanner_graphs() @@ -61,7 +61,7 @@ class GraphWriter(WebsiteWriter): + " }\n" + " .graphbox {\n" + " text-align: center;\n" - #+ " display: none;\n" + + " display: none;\n" + " }\n" + " .faravahar {\n" + " fill: none;\n" @@ -144,10 +144,8 @@ class GraphWriter(WebsiteWriter): + " <span class=\"maatuska\" style=\"margin-left:5px\">     </span> Maatuska\n" + " <span class=\"longclaw\" style=\"margin-left:5px\">     </span> Longclaw\n" + " <span class=\"tor26\" style=\"margin-left:5px\">     </span> tor26\n" - + " <span class=\"urras\" style=\"margin-left:5px\">     </span> urras\n" + " <span class=\"dizum\" style=\"margin-left:5px\">     </span> dizum\n" + " <span class=\"dannenberg\" style=\"margin-left:5px\">     </span> dannenberg\n" - + " <span class=\"turtles\" style=\"margin-left:5px\">     </span> turtles\n" + " </div>\n" + " </td>\n" + " </tr>\n") @@ -165,7 +163,14 @@ class GraphWriter(WebsiteWriter): + " <colgroup>\n" + " <col width=\"160\">\n" + " <col width=\"640\">\n" - + " </colgroup>\n") + + " </colgroup>\n" + + " <tr class=\"graphplaceholder\">\n" + + " <td>\n" + + " <div style=\"text-align:center\">\n" + + " Generating Graph... (requires SVG and Javascript support)\n" + + " </div>\n" + + " </td>\n" + + " </tr>\n") self._write_number_of_relays_voted_about_graphs_spot("voted_total_1") self._write_number_of_relays_voted_about_graphs_spot("voted_total_2") self._write_number_of_relays_voted_about_graphs_spot("voted_total_3") @@ -207,7 +212,14 @@ class GraphWriter(WebsiteWriter): + " <colgroup>\n" + " <col width=\"160\">\n" + " <col width=\"640\">\n" - + " </colgroup>\n") + + " </colgroup>\n" + + " <tr class=\"graphplaceholder\">\n" + + " <td>\n" + + " <div style=\"text-align:center\">\n" + + " Generating Graph... (requires SVG and Javascript support)\n" + + " </div>\n" + + " </td>\n" + + " </tr>\n") self._write_bandwidth_scanner_graphs_spot("bwauth_measured_1") self._write_bandwidth_scanner_graphs_spot("bwauth_measured_2") self._write_bandwidth_scanner_graphs_spot("bwauth_measured_3") @@ -398,6 +410,17 @@ class GraphWriter(WebsiteWriter): .attr("class", "graph-title") .text(graph.title); } + + var toShow = document.getElementsByClassName('graphbox'); + for(i=0; i<toShow.length; i++) { + console.log(toShow[i]); + toShow[i].style.display = 'block'; + } + var toHide = document.getElementsByClassName('graphplaceholder'); + for(i=0; i<toHide.length; i++) { + console.log(toHide[i]); + toHide[i].style.display = 'none'; + } }); </script>""" diff --git a/website.py b/website.py index 569c8a1..574ede9 100755 --- a/website.py +++ b/website.py @@ -27,7 +27,7 @@ class WebsiteWriter: self._write_valid_after_time() self._write_signatures() self._write_known_flags() - self._write_number_of_relays_voted_about() + self._write_number_of_relays_voted_about(True) self._write_consensus_methods() self._write_recommended_versions() self._write_consensus_parameters() @@ -228,7 +228,7 @@ class WebsiteWriter: + "</table>\n") #----------------------------------------------------------------------------------------- - def _write_number_of_relays_voted_about(self): + def _write_number_of_relays_voted_about(self, linkToGraph): """ Write the number of relays voted about. """ @@ -236,9 +236,14 @@ class WebsiteWriter: + " " + "<a name=\"numberofrelays\">\n" + "<h3><a href=\"#numberofrelays\" class=\"anchor\">" - + "Number of relays voted about</a></h3>\n" - + "<br>\n" - + "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n" + + "Number of relays voted about</a></h3>\n") + if linkToGraph: + self.site.write("<p>\n" + + " You can also view <a href=\"graphs.html\">historical voting graphs</a>.\n" + + "</p>\n") + else: + self.site.write("<br />\n") + self.site.write("<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n" + " <colgroup>\n" + " <col width=\"160\">\n" + " <col width=\"320\">\n" diff --git a/write_website.py b/write_website.py index 25781b0..87863d6 100755 --- a/write_website.py +++ b/write_website.py @@ -95,7 +95,7 @@ def main(): dbc.commit() # Write out the updated csv file for the graphs - vote_data = dbc.execute("SELECT * from vote_data ORDER BY date DESC LIMIT 17520") + vote_data = dbc.execute("SELECT * from vote_data ORDER BY date DESC LIMIT 2160") f = open(os.path.join(os.path.dirname(__file__), 'out', 'vote-stats.csv'), 'w') f.write("date") for d in get_dirauths_in_tables():

1 0

[depictor/master] Add the vote data for July 5/6
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit 232083c2bc63dbad383fc244c3c2c6c82820bf10 Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Jul 6 13:41:34 2016 -0400 Add the vote data for July 5/6 --- data/historical.db | Bin 2191360 -> 2191360 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/data/historical.db b/data/historical.db index fd4b938..6073169 100644 Binary files a/data/historical.db and b/data/historical.db differ

1 0

[depictor/master] Accidently was getting the javascript externally
by tom＠torproject.org 01 Sep '16

01 Sep '16

commit 2aa72ed90ffdee733218a79472b9bd8a98e8bd80 Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Jul 6 13:12:54 2016 -0500 Accidently was getting the javascript externally --- graphs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphs.py b/graphs.py index e36ca45..176ec27 100755 --- a/graphs.py +++ b/graphs.py @@ -42,7 +42,7 @@ class GraphWriter(WebsiteWriter): + " <meta http-equiv=\"content-type\" content=\"text/html; charset=ISO-8859-1\">\n" + " <link href=\"stylesheet-ltr.css\" type=\"text/css\" rel=\"stylesheet\">\n" + " <link href=\"favicon.ico\" type=\"image/x-icon\" rel=\"shortcut icon\">\n" - + " <script src=\"https://d3js.org/d3.v4.min.js\"></script>\n" + + " <script src=\"d3.v4.min.js\"></script>\n" + " </head>\n" + " <body>\n" + " <style>\n"

1 0

[tor/master] Fix memory leaks in the shared random tests.
by nickm＠torproject.org 01 Sep '16

01 Sep '16

commit 6c1d1127a84b019d1100d38f083f19022b933eb7 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 1 09:10:27 2016 -0400 Fix memory leaks in the shared random tests. Please remember to test your code with --enable-expensive-hardening. :) --- src/test/test_shared_random.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 8368300..ead8d03 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -274,6 +274,7 @@ test_sr_commit(void *arg) time_t now = time(NULL); sr_commit_t *our_commit = NULL; smartlist_t *args = smartlist_new(); + sr_commit_t *parsed_commit = NULL; (void) arg; @@ -340,13 +341,12 @@ test_sr_commit(void *arg) /* We'll build a list of values from our commit that our parsing function * takes from a vote line and see if we can parse it correctly. */ { - sr_commit_t *parsed_commit; smartlist_add(args, tor_strdup("1")); smartlist_add(args, tor_strdup(crypto_digest_algorithm_get_name(our_commit->alg))); smartlist_add(args, tor_strdup(sr_commit_get_rsa_fpr(our_commit))); - smartlist_add(args, our_commit->encoded_commit); - smartlist_add(args, our_commit->encoded_reveal); + smartlist_add(args, tor_strdup(our_commit->encoded_commit)); + smartlist_add(args, tor_strdup(our_commit->encoded_reveal)); parsed_commit = sr_parse_commit(args); tt_assert(parsed_commit); /* That parsed commit should be _EXACTLY_ like our original commit (we @@ -354,15 +354,14 @@ test_sr_commit(void *arg) parsed_commit->valid = 1; tt_mem_op(parsed_commit, OP_EQ, our_commit, sizeof(*parsed_commit)); /* Cleanup */ - tor_free(smartlist_get(args, 0)); /* strdup here. */ - tor_free(smartlist_get(args, 1)); /* strdup here. */ - smartlist_clear(args); - sr_commit_free(parsed_commit); } done: + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); sr_commit_free(our_commit); + sr_commit_free(parsed_commit); + authority_cert_free(auth_cert); } /* Test the encoding and decoding function for commit and reveal values. */ @@ -583,6 +582,7 @@ test_vote(void *arg) smartlist_free(tokens); smartlist_clear(args); smartlist_free(args); + tor_free(lines); } done: @@ -780,7 +780,7 @@ test_sr_setup_commits(void) tt_assert(!commit_has_reveal_value(commit_d)); done: - return; + authority_cert_free(auth_cert); } /** Verify that the SRV generation procedure is proper by testing it against @@ -967,6 +967,7 @@ test_utils(void *arg) /* Change the pubkey. */ memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity)); tt_int_op(commit_is_authoritative(&commit, digest), ==, 0); + crypto_pk_free(k); } /* Testing get_phase_str(). */ @@ -1044,6 +1045,7 @@ test_state_transition(void *arg) prev = sr_state_get_previous_srv(); tt_assert(prev == cur); tt_assert(!sr_state_get_current_srv()); + sr_state_clean_srvs(); } /* New protocol run. */ @@ -1092,14 +1094,16 @@ test_keep_commit(void *arg) sr_commit_t *commit = NULL, *dup_commit = NULL; sr_state_t *state; time_t now = time(NULL); + crypto_pk_t *k = NULL; (void) arg; MOCK(trusteddirserver_get_by_v3_auth_digest, trusteddirserver_get_by_v3_auth_digest_m); - { /* Setup a minimal dirauth environment for this test */ - crypto_pk_t *k = crypto_pk_new(); + { + k = crypto_pk_new(); + /* Setup a minimal dirauth environment for this test */ /* Have a key that is not the one from our commit. */ tt_int_op(0, ==, crypto_pk_generate_key(k)); tt_int_op(0, ==, crypto_pk_get_fingerprint(k, fp, 0)); @@ -1176,6 +1180,7 @@ test_keep_commit(void *arg) done: sr_commit_free(commit); sr_commit_free(dup_commit); + crypto_pk_free(k); UNMOCK(trusteddirserver_get_by_v3_auth_digest); }

1 0