March 2015 - tor-commits - lists.torproject.org

[depictor/master] Update the License with my name and new years.
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit 1743b81b4c2425e5a88bf03d2f82ed9ec3e408ed Author: Tom Ritter <tom(a)ritter.vg> Date: Tue Mar 3 14:11:12 2015 +0100 Update the License with my name and new years. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 338d619..c38a97d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013, Damian Johnson and The Tor Project +Copyright (c) 2013,14,15 Damian Johnson, Tom Ritter, and The Tor Project All rights reserved. Redistribution and use in source and binary forms, with or without

1 0

[depictor/master] Link each relay to itself in the previous
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit 93b83263743555f5b1f20df803288e54ac0a8f50 Author: Tom Ritter <tom(a)ritter.vg> Date: Thu Mar 5 00:46:07 2015 +0100 Link each relay to itself in the previous --- website.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website.py b/website.py index 2f8c45c..3b95a57 100755 --- a/website.py +++ b/website.py @@ -865,15 +865,13 @@ class WebsiteWriter: if relay_fp in self.consensus.routers and \ "Named" in self.consensus.routers[relay_fp].flags and \ relay_nickname[0].isdigit(): - self.site.write(" <td id=\"" \ - + relay_nickname \ - + "\">" \ + self.site.write(" <td id=\"" + relay_fp + "\">" \ + relay_fp.substring(0, 8) \ + " " \ + relay_fp \ + "</td>\n") else: - self.site.write(" <td>" \ + self.site.write(" <td id=\"" + relay_fp + "\">" \ + relay_fp[0:8] + " " \ + relay_fp @@ -884,7 +882,9 @@ class WebsiteWriter: + " <a href=\"https://atlas.torproject.org/#details/" \ + relay_fp + "\">Atlas</a> | " \ + "<a href=\"https://globe.torproject.org/#/relay/" + relay_fp \ - + "\">Globe</a>" \ + + "\">Globe</a> | <a href=\"consensus-health-" \ + + (self.get_consensus_time() - datetime.timedelta(hours=1)).strftime("%Y-%m-%d-%H-%M") + + ".html#" + relay_fp + "\">⇜</a>" \ + "</td>\n") relevantFlags = set()

1 0

[depictor/master] Highlight only the non-matching parameter
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit f04375a1a7afc822bd6593539808dc71155a6dc7 Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Mar 4 17:36:26 2015 +0100 Highlight only the non-matching parameter --- website.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/website.py b/website.py index baca4e7..fbdf4ee 100755 --- a/website.py +++ b/website.py @@ -421,23 +421,25 @@ class WebsiteWriter: for dirauth_nickname in self.known_authorities: if dirauth_nickname in self.votes: vote = self.votes[dirauth_nickname] - #XXX - Only Write invalid/odd parameter in red - conflictOrInvalid = False + conflictOrInvalid = [] if vote.params: for p in vote.params: if (p not in self.known_params and not p.startswith('bwauth')) or \ p not in self.consensus.params or \ self.consensus.params[p] != vote.params[p]: - conflictOrInvalid = True + conflictOrInvalid.append(p) break if conflictOrInvalid: self.site.write(" <tr>\n" + " <td>" + dirauth_nickname + "</td>\n" - + " <td>params") + + " <td>params") for p in vote.params: - self.site.write(" " + p + "=" + str(vote.params[p])) - self.site.write("</td>\n" + if p in conflictOrInvalid: + self.site.write(" " + p + "=" + str(vote.params[p]) + "") + else: + self.site.write(" " + p + "=" + str(vote.params[p])) + self.site.write("</td>\n" + " </tr>\n") else: self.site.write(" <tr>\n"

1 0

[depictor/master] print a 'missing' line if a BWAuth is missing.
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit c206e542d28ec5748915a17efc3f13be3c377a47 Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Mar 4 20:01:19 2015 +0100 print a 'missing' line if a BWAuth is missing. --- website.py | 15 +++++++++++---- write_website.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/website.py b/website.py index 8c4fd0d..2f8c45c 100755 --- a/website.py +++ b/website.py @@ -52,8 +52,9 @@ class WebsiteWriter: self.consensus_expirey = timedelta def set_directory_key_warning_time(self, timedelta): self.directory_key_warning_time = timedelta - def set_known_params(self, kp): - self.known_params = kp + def set_config(self, config): + self.known_params = config['known_params'] + self.bandwidth_authorities = config['bandwidth_authorities'] def get_consensus_time(self): return self.consensus.valid_after @@ -533,7 +534,6 @@ class WebsiteWriter: if not self.votes: self.site.write(" <tr><td>(No votes.)</td><td></td></tr>\n") else: - #XXX - loop over all bwauths, and print if vote is not present for dirauth_nickname in self.votes: vote = self.votes[dirauth_nickname] @@ -548,6 +548,13 @@ class WebsiteWriter: + " <td>" + str(bandwidthWeights) + " Measured values in w lines</td>\n" + " </tr>\n") + for dirauth_nickname in self.bandwidth_authorities: + if dirauth_nickname not in self.votes: + self.site.write(" <tr>\n" + + " <td>" + dirauth_nickname + "</td>\n" + + " <td class=\"oiv\">Missing vote</td>\n" + + " </tr>\n") + self.site.write("</table>\n") #----------------------------------------------------------------------------------------- @@ -971,7 +978,7 @@ if __name__ == '__main__': }) config = stem.util.conf.get_config("consensus") config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg')) - w.set_known_params(CONFIG['known_params']) + w.set_config(CONFIG) w.write_website(os.path.join(os.path.dirname(__file__), 'out', \ 'consensus-health-' + w.get_consensus_time().strftime("%Y-%m-%d-%H-%M") + '.html'), True) diff --git a/write_website.py b/write_website.py index 3e6a500..99e18c5 100755 --- a/write_website.py +++ b/write_website.py @@ -63,7 +63,7 @@ def main(): w = WebsiteWriter() w.set_consensuses(consensuses) w.set_votes(votes) - w.set_known_params(CONFIG['known_params']) + w.set_config(CONFIG) w.write_website(os.path.join(os.path.dirname(__file__), 'out', \ 'consensus-health-' + w.get_consensus_time().strftime("%Y-%m-%d-%H-%M") + '.html'), True) w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'consensus-health.html'), True)

1 0

[depictor/master] Print Identity and Signing Key Lengths
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit cbd243e81813ad4f7df94191b7abc28ddabdd31e Author: Tom Ritter <tom(a)ritter.vg> Date: Sun Mar 8 13:43:35 2015 -0500 Print Identity and Signing Key Lengths --- website.py | 48 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/website.py b/website.py index 3b95a57..d33d3fc 100755 --- a/website.py +++ b/website.py @@ -10,6 +10,8 @@ import os import time import operator import datetime +from base64 import b64decode +from Crypto.PublicKey import RSA class WebsiteWriter: consensus = None @@ -43,7 +45,7 @@ class WebsiteWriter: def set_consensuses(self, c): self.consensuses = c self.consensus = max(c.itervalues(), key=operator.attrgetter('valid_after')) - # XXX - Change this to be a list of known dir auths, don't calculate it off the consensus because sometimes they're missing entirely + # XXX - Change this to be a list of known dir auths, don't calculate it off the consensus because sometimes they're missing entirely self.known_authorities = set([r.nickname for r in self.consensus.routers.values() if 'Authority' in r.flags and r.nickname != "Tonga"]) self.known_authorities.update([r.nickname for r in self.consensus.directory_authorities]) def set_votes(self, v): @@ -150,7 +152,6 @@ class WebsiteWriter: + " <col width=\"480\">\n" + " </colgroup>\n") - # XXX Should also write the keysize here # XXX Should also write if the displayed consensus is out of date signingFPs = {sig.identity:sig.method for sig in self.consensus.signatures} for dirauth_nickname in self.known_authorities: @@ -468,6 +469,14 @@ class WebsiteWriter: + "</table>\n") #----------------------------------------------------------------------------------------- + def getKeySize(self, key): + key = key.replace("-----BEGIN RSA PUBLIC KEY-----", "") + key = key.replace("-----END RSA PUBLIC KEY-----", "") + key = key.replace("\n", "") + keyDER = b64decode(key) + keyPub = RSA.importKey(keyDER) + return str(keyPub.size() + 1) + def _write_authority_keys(self): """ Write authority keys and their expiration dates. @@ -481,8 +490,17 @@ class WebsiteWriter: + "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n" + " <colgroup>\n" + " <col width=\"160\">\n" - + " <col width=\"640\">\n" - + " </colgroup>\n") + + " <col width=\"300\">\n" + + " <col width=\"170\">\n" + + " <col width=\"170\">\n" + + " </colgroup>\n" + + " <tr>\n" + + " <th>Name</th>" + + " <th>Expirey</th>" + + " <th>Identity Key Len.</th>" + + " <th>Signing Key Len.</th>" + + " </tr>\n") + if not self.votes: self.site.write(" <tr><td>(No votes.)</td><td></td></tr>\n") else: @@ -493,25 +511,27 @@ class WebsiteWriter: voteDirKeyExpires = vote.directory_authorities[0].key_certificate.expires if voteDirKeyExpires - self.directory_key_warning_time < datetime.datetime.now(): self.site.write(" <tr>\n" - + " <td>" + dirauth_nickname + "</td>\n" - + " <td>dir-key-expires " - + voteDirKeyExpires.isoformat().replace("T", " ") + "</td>\n" - + " </tr>\n"); + + " <td>" + dirauth_nickname + "</td>\n" + + " <td>" + + voteDirKeyExpires.isoformat().replace("T", " ") + "</td>\n") else: self.site.write(" <tr>\n" - + " <td>" + dirauth_nickname + "</td>\n" - + " <td>dir-key-expires " - + voteDirKeyExpires.isoformat().replace("T", " ") + "</td>\n" - + " </tr>\n"); + + " <td>" + dirauth_nickname + "</td>\n" + + " <td>" + + voteDirKeyExpires.isoformat().replace("T", " ") + "</td>\n") + + self.site.write(" <td>" + self.getKeySize(vote.directory_authorities[0].key_certificate.identity_key) + "</td>\n" + + " <td>" + self.getKeySize(vote.directory_authorities[0].key_certificate.signing_key) + "</td>\n" + + " </tr>\n"); else: self.site.write(" <tr>\n" + " <td>" + dirauth_nickname + "</td>\n" - + " <td>Vote Not Present</td>\n" + + " <td colspan=\"3\">Vote Not Present</td>\n" + " </tr>\n") self.site.write("</table>\n" + " \n" - + "Note that expiration dates of legacy keys are " + + "Note that expiration dates of any legacy keys are " + "not included in votes and therefore not listed here!" + "\n")

1 0

[depictor/master] Add links to Atlas and Globe
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit 8dddb1dd4d645bc0a52c7d195f881571d7c3eea5 Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Mar 4 15:34:05 2015 +0100 Add links to Atlas and Globe --- out/stylesheet-ltr.css | 4 ++++ website.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/out/stylesheet-ltr.css b/out/stylesheet-ltr.css index 77bd55f..8ace3a9 100644 --- a/out/stylesheet-ltr.css +++ b/out/stylesheet-ltr.css @@ -98,6 +98,10 @@ td { font-size: 3px; } +.agt { + font-size: .8em; +} + /* the banner */ table.banner { diff --git a/website.py b/website.py index b5db508..baca4e7 100755 --- a/website.py +++ b/website.py @@ -868,7 +868,13 @@ class WebsiteWriter: + relay_fp + "</td>\n") - self.site.write(" <td>" + relay_nickname + "</td>\n") + self.site.write(" <td>" \ + + relay_nickname \ + + " <a href=\"https://atlas.torproject.org/#details/" \ + + relay_fp + "\">Atlas</a> | " \ + + "<a href=\"https://globe.torproject.org/#/relay/" + relay_fp \ + + "\">Globe</a>" \ + + "</td>\n") relevantFlags = set() for dirauth_nickname in self.votes:

1 0

[depictor/master] gzip the archival version of consensus-health.html
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit 932b67dfde560fcc1739fe200040c39a5368b986 Author: Tom Ritter <tom(a)ritter.vg> Date: Sun Mar 8 11:46:24 2015 -0500 gzip the archival version of consensus-health.html --- write_website.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/write_website.py b/write_website.py index 99e18c5..c0deac6 100755 --- a/write_website.py +++ b/write_website.py @@ -12,6 +12,7 @@ import time import datetime import operator import traceback +import subprocess import stem.descriptor import stem.descriptor.remote @@ -64,11 +65,14 @@ def main(): w.set_consensuses(consensuses) w.set_votes(votes) w.set_config(CONFIG) - w.write_website(os.path.join(os.path.dirname(__file__), 'out', \ - 'consensus-health-' + w.get_consensus_time().strftime("%Y-%m-%d-%H-%M") + '.html'), True) w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'consensus-health.html'), True) w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'index.html'), False) + archived = os.path.join(os.path.dirname(__file__), 'out', \ + 'consensus-health-' + w.get_consensus_time().strftime("%Y-%m-%d-%H-%M") + '.html') + subprocess.call(["cp", os.path.join(os.path.dirname(__file__), 'out', 'consensus-health.html'), archived]) + subprocess.call(["gzip", "-9", archived]) + def get_consensuses(): """

1 0

[depictor/master] Add commented debugging to write_website.py
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit b0f88d0eab7dc04bb5eac7fa3ca12e204feb2b2d Author: Tom Ritter <tom(a)ritter.vg> Date: Sun Mar 8 12:58:27 2015 -0500 Add commented debugging to write_website.py --- write_website.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/write_website.py b/write_website.py index c0deac6..415c7ea 100755 --- a/write_website.py +++ b/write_website.py @@ -59,8 +59,12 @@ def main(): f.write("%s,%i,%i\n" % (ds, time.time() * 1000, int(consensus_fetching_runtimes[ds] * 1000))) f.close() - # produces the website + # great for debugging + #import pickle + #pickle.dump(consensuses, open('consensus.p', 'wb')) + #pickle.dump(votes, open('votes.p', 'wb')) + # produces the website w = WebsiteWriter() w.set_consensuses(consensuses) w.set_votes(votes)

1 0

[depictor/master] Update the debugging pattern and method
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit bf3715500c98241fc955bd874d1a9416d6f1b111 Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Mar 4 17:54:19 2015 +0100 Update the debugging pattern and method --- website.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/website.py b/website.py index 2de8cc2..8c4fd0d 100755 --- a/website.py +++ b/website.py @@ -947,18 +947,33 @@ class WebsiteWriter: if __name__ == '__main__': """ I found that the most effective way to test this independently was to pickle the - downloaded conensuses in ./consensus_health_checker.py like this: + downloaded conensuses in ./write_website.py like this: import pickle pickle.dump(consensuses, open('consensus.p', 'wb')) pickle.dump(votes, open('votes.p', 'wb')) - Then I can run ./website.pt and pdb.set_trace() where needed to debug + Then I can run ./website.py and pdb.set_trace() where needed to debug """ + import stem import pickle w = WebsiteWriter() + c = pickle.load(open('consensus.p', 'rb')) - w.set_consensus(c.values()[0])#Just take one of them + w.set_consensuses(c) v = pickle.load(open('votes.p', 'rb')) w.set_votes(v) - w.write_website('consensus-health-test.html') + + CONFIG = stem.util.conf.config_dict('consensus', { + 'ignored_authorities': [], + 'bandwidth_authorities': [], + 'known_params': [], + }) + config = stem.util.conf.get_config("consensus") + config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg')) + w.set_known_params(CONFIG['known_params']) + + w.write_website(os.path.join(os.path.dirname(__file__), 'out', \ + 'consensus-health-' + w.get_consensus_time().strftime("%Y-%m-%d-%H-%M") + '.html'), True) + w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'consensus-health.html'), True) + w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'index.html'), False) \ No newline at end of file

1 0

[depictor/master] Write a historical version
by tom＠torproject.org 23 Mar '15

23 Mar '15

commit 4ef382841c2e2fe1057f36cbb4beab2559c74f5f Author: Tom Ritter <tom(a)ritter.vg> Date: Wed Mar 4 17:53:12 2015 +0100 Write a historical version --- .gitignore | 2 +- website.py | 2 ++ write_website.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e54e242..ad4f990 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ *.swp stem out/index.html -out/consensus-health.html +out/consensus-health* out/download-stats.csv \ No newline at end of file diff --git a/website.py b/website.py index fbdf4ee..2de8cc2 100755 --- a/website.py +++ b/website.py @@ -54,6 +54,8 @@ class WebsiteWriter: self.directory_key_warning_time = timedelta def set_known_params(self, kp): self.known_params = kp + def get_consensus_time(self): + return self.consensus.valid_after #----------------------------------------------------------------------------------------- def _write_page_header(self, include_relay_flags): diff --git a/write_website.py b/write_website.py index f445a8e..3e6a500 100755 --- a/write_website.py +++ b/write_website.py @@ -64,6 +64,8 @@ def main(): w.set_consensuses(consensuses) w.set_votes(votes) w.set_known_params(CONFIG['known_params']) + w.write_website(os.path.join(os.path.dirname(__file__), 'out', \ + 'consensus-health-' + w.get_consensus_time().strftime("%Y-%m-%d-%H-%M") + '.html'), True) w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'consensus-health.html'), True) w.write_website(os.path.join(os.path.dirname(__file__), 'out', 'index.html'), False)

1 0