commit f8c25e7efd545a6e00672dce76070a2bd489494e Author: Tom Ritter <tom@ritter.vg> Date: Mon Jan 27 15:25:49 2020 +0100 Update to python3 and a bump to stem master --- graphs.py | 8 ++++---- utility.py | 40 ++++++++++++++++++++++++---------------- website.py | 29 ++++++++++++++++------------- write_website.py | 22 +++++++++++----------- 4 files changed, 55 insertions(+), 44 deletions(-) diff --git a/graphs.py b/graphs.py index 695d9b6..5d60074 100755 --- a/graphs.py +++ b/graphs.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # See LICENSE for licensing information """ -Produces an HTML file for easily viewing voting and consensus differences +processoduces an HTML file for easily viewing voting and consensus differences Ported from Java version Doctor """ @@ -358,8 +358,8 @@ class GraphWriter(WebsiteWriter): - var bwauths = """ + str(get_bwauths().keys()) + """; - var dirauths = """ + str(get_dirauths().keys()) + """; + var bwauths = """ + str(list(get_bwauths().keys())) + """; + var dirauths = """ + str(list(get_dirauths().keys())) + """; var ignore_fallback_dirs = """ + str(self.config['ignore_fallback_authorities']).lower() + """; var _getBandwidthDataValue = function(d, dirauth) { return d[dirauth + "_bwauth"]; } diff --git a/utility.py b/utility.py index 6f4f9aa..7b35f00 100755 --- a/utility.py +++ b/utility.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import time import urllib @@ -9,22 +9,26 @@ import stem.descriptor.remote import stem.util.conf import stem.util.enum -from stem.util.lru_cache import lru_cache - config = {'bwauths': []} def set_config(c): global config config = c -@lru_cache() +_dirAuths = None def get_dirauths(): - #Remove any BridgeAuths - return dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.v3ident) + global _dirAuths + if _dirAuths == None: + #Remove any BridgeAuths + _dirAuths = dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.v3ident) + return _dirAuths -@lru_cache() +_bwAuths = None def get_bwauths(): global config - return dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.nickname.lower() in config['bwauths']) + global _bwAuths + if _bwAuths == None: + _bwAuths = dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.nickname.lower() in config['bwauths']) + return _bwAuths # How to grab a vote or consensus with stem: """ @@ -80,7 +84,7 @@ def _get_documents(label, resource): start_time = time.time() documents[nickname] = query.run()[0] runtimes[nickname] = time.time() - start_time - except Exception, exc: + except Exception as exc: if label == 'vote': # try to download the vote via the other authorities @@ -104,20 +108,24 @@ def _get_documents(label, resource): def get_clockskew(): clockskew = {} for (nickname, authority) in get_dirauths().items(): - authority_address = "http://" + str(authority.address) + ":" + str(authority.dir_port) + authority_address = "http://" + str(authority.address) + ":" + str(authority.dir_port) + "/tor/keys/authority.z" try: startTimeStamp = datetime.datetime.utcnow() startTime = time.time() - f = urllib.urlopen(authority_address) - for h in f.info().headers: - if h.upper().startswith('DATE:'): - clockskew[nickname] = datetime.datetime.strptime(h[6:].strip(), '%a, %d %b %Y %H:%M:%S %Z') + f = urllib.request.urlopen(authority_address) + h = f.getheader('date') + if h: + clockskew[nickname] = datetime.datetime.strptime(h, '%a, %d %b %Y %H:%M:%S %Z') + else: + print("Could not get clockskew for ", nickname) + continue processing = time.time() - startTime if processing > 5: clockskew[nickname] -= datetime.timedelta(seconds=(processing / 2)) clockskew[nickname] -= startTimeStamp clockskew[nickname] = clockskew[nickname].total_seconds() - except: + except Exception as e: + print("Clockskew Exception:", e) continue return clockskew @@ -142,4 +150,4 @@ class FileMock(): if __name__ == "__main__": skew = get_clockskew() for c in skew: - print c, skew[c] \ No newline at end of file + print(c, skew[c]) \ No newline at end of file diff --git a/website.py b/website.py index 455e0e1..2f5d299 100755 --- a/website.py +++ b/website.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # See LICENSE for licensing information """ @@ -71,7 +71,7 @@ class WebsiteWriter: def set_consensuses(self, c): self.consensuses = c - self.consensus = max(c.itervalues(), key=operator.attrgetter('valid_after')) + self.consensus = max(c.values(), key=operator.attrgetter('valid_after')) self.known_authorities = get_dirauths().keys() self.bandwidth_authorities = get_bwauths().keys() def set_votes(self, v): @@ -1053,7 +1053,7 @@ class WebsiteWriter: bandwidthWeights = 0 for r in vote.routers.values(): - if r.measured >= 0L: + if r.measured and r.measured >= int(0): bandwidthWeights += 1 if bandwidthWeights > 0: @@ -1223,7 +1223,7 @@ class WebsiteWriter: maxDownloadsForAnyAuthority = max(len(downloadData[a]), maxDownloadsForAnyAuthority) def getPercentile(dataset, percentile): - index = (percentile * (len(dataset) - 1)) / 100 + index = int((percentile * (len(dataset) - 1)) / 100) return str(dataset[index]) self.site.write("<br>\n\n\n" @@ -1522,7 +1522,7 @@ class WebsiteWriter: allRelays[relay_fp] = self.consensus.routers[relay_fp].nickname linesWritten = 0 - sortedKeys = allRelays.keys() + sortedKeys = list(allRelays.keys()) sortedKeys.sort() for relay_fp in sortedKeys: if linesWritten % 10 == 0: @@ -1566,9 +1566,10 @@ class WebsiteWriter: bwauths_voted = 0 for dirauth_nickname in self.votes: if relay_fp in self.votes[dirauth_nickname].routers: - if self.votes[dirauth_nickname].routers[relay_fp].measured >= 0L: + measured = self.votes[dirauth_nickname].routers[relay_fp].measured + if measured and measured >= int(0): bwauths_voted += 1 - if target_bw == self.votes[dirauth_nickname].routers[relay_fp].measured: + if target_bw == measured: bwauths.append(dirauth_nickname) if len(bwauths) == bwauths_voted: return ["all"] @@ -1640,9 +1641,10 @@ class WebsiteWriter: elif consensusFlags and flag in vote.known_flags and flag in consensusFlags: self.site.write( "<span class=\"oict\">!</span><span class=\"oic\">" + flag + "</span>") - if vote.routers[relay_fp].measured >= 0L: + measured = vote.routers[relay_fp].measured + if measured and measured >= int(0): self.site.write(" <br />" if flagsWritten > 0 else "") - self.site.write("bw=" + str(vote.routers[relay_fp].measured)) + self.site.write("bw=" + str(measured)) flagsWritten += 1 self.site.write("</td>\n"); @@ -1660,9 +1662,10 @@ class WebsiteWriter: if flag in consensusFlags: self.site.write(flag) - if self.consensus.routers[relay_fp].bandwidth >= 0L: + bandwidth = self.consensus.routers[relay_fp].bandwidth + if bandwidth and bandwidth >= int(0): self.site.write(" <br />" if flagsWritten > 0 else "") - self.site.write("bw=" + str(self.consensus.routers[relay_fp].bandwidth)) + self.site.write("bw=" + str(bandwidth)) flagsWritten += 1 if not self.consensus.routers[relay_fp].is_unmeasured: assigning_bwauths = self.__find_assigning_bwauth_for_bw_value(relay_fp) @@ -1707,8 +1710,8 @@ class WebsiteWriter: + "<div class=\"bottom\" id=\"bottom\">\n" + "<p>This page was generated with <a href=\"" + "https://gitweb.torproject.org/depictor.git/\">depictor</a> version " - + depictor_version + " and <a href=\"https://gitweb.torproject.org/stem.git/" - + "\">stem</a> version " + stem_version + "</p>" + + str(depictor_version) + " and <a href=\"https://gitweb.torproject.org/stem.git/" + + "\">stem</a> version " + str(stem_version) + "</p>" + "<p>\"Tor\" and the \"Onion Logo\" are <a " + "href=\"https://www.torproject.org/docs/trademark-faq.html.en\">" + "registered trademarks</a> of The Tor Project, Inc.</p>\n" diff --git a/write_website.py b/write_website.py index 538c10c..4f6224e 100755 --- a/write_website.py +++ b/write_website.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2013, Damian Johnson, Tom Ritter, and The Tor Project # See LICENSE for licensing information @@ -100,7 +100,7 @@ def main(): if expected == ut_to_datetime(d): pass else: - print "We seem to be missing", consensus_datetime_format(expected) + print("We seem to be missing", consensus_datetime_format(expected)) dbc.execute("INSERT OR REPLACE INTO " + tbl + "(date) VALUES (?)", (unix_time(expected),)) dbc.commit() previous = d @@ -109,13 +109,13 @@ def main(): if not CONFIG['ignore_fallback_authorities']: fallback_dirs_running = 0 fallback_dirs_notrunning = 0 - for relay_fp in consensuses.values()[0].routers: - if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags: + for relay_fp in list(consensuses.values())[0].routers: + if relay_fp in fallback_dirs and 'Running' in list(consensuses.values())[0].routers[relay_fp].flags: fallback_dirs_running += 1 elif relay_fp in fallback_dirs: fallback_dirs_notrunning += 1 - insertValues = [unix_time(consensuses.values()[0].valid_after)] + insertValues = [unix_time(list(consensuses.values())[0].valid_after)] insertValues.append(fallback_dirs_running) insertValues.append(fallback_dirs_notrunning) insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning) @@ -148,7 +148,7 @@ def main(): runningRelays = 0 bandwidthWeights = 0 for r in vote.routers.values(): - if r.measured >= 0L: + if r.measured and r.measured >= int(0): bandwidthWeights += 1 if u'Running' in r.flags: runningRelays += 1 @@ -159,7 +159,7 @@ def main(): for c in vote_data_schema: vote_data_columns.add(c[1].replace("_known", "").replace("_running", "").replace("_bwauth", "").lower()) - insertValues = [unix_time(consensuses.values()[0].valid_after)] + insertValues = [unix_time(list(consensuses.values())[0].valid_after)] createColumns = "" insertColumns = "date" insertQuestions = "" @@ -209,7 +209,7 @@ def main(): data[dirauth_nickname] = {'unmeasured' : 0, 'above' : 0, 'below' : 0, 'exclusive' : 0 , 'shared' : 0} had_any_value = False - for r in consensuses.values()[0].routers.values(): + for r in list(consensuses.values())[0].routers.values(): if r.is_unmeasured: continue elif r.fingerprint not in vote.routers or vote.routers[r.fingerprint].measured == None: @@ -229,7 +229,7 @@ def main(): had_any_value = True data[dirauth_nickname]['shared'] += 1 else: - print "What case am I in???" + print("What case am I in???") sys.exit(1) if not had_any_value: @@ -240,7 +240,7 @@ def main(): for c in bwauth_stats_data_schema: bwauth_stats_data_columns.add(c[1].replace("_above", "").replace("_shared", "").replace("_exclusive", "").replace("_below", "").replace("_unmeasured", "").lower()) - insertValues = [unix_time(consensuses.values()[0].valid_after)] + insertValues = [unix_time(list(consensuses.values())[0].valid_after)] createColumns = "" insertColumns = "date" insertQuestions = "" @@ -343,4 +343,4 @@ if __name__ == '__main__': main() except: msg = "%s failed with:\n\n%s" % (sys.argv[0], traceback.format_exc()) - print "Error: %s" % msg + print("Error: %s" % msg)