January 2020 - tor-commits - lists.torproject.org

[fallback-scripts/master] update: py3: Skip lazy evaluation where needed
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 549334ea388359e7ec28013e07186bb986dd2474 Author: teor <teor(a)torproject.org> Date: Mon Dec 2 16:06:45 2019 +1000 update: py3: Skip lazy evaluation where needed Part of 28863. --- updateFallbackDirs.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index e97a501..a0c8ad6 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -967,7 +967,7 @@ class Candidate(object): generic_history = [] - periods = history.keys() + periods = list(history.keys()) periods.sort(key = lambda x: history[x]['interval']) now = datetime.datetime.utcnow() newest = now @@ -1685,9 +1685,9 @@ class CandidateList(dict): # Find fallbacks that fit the uptime, stability, and flags criteria, # and make an array of them in self.fallbacks def compute_fallbacks(self): - self.fallbacks = map(lambda x: self[x], - filter(lambda x: self[x].is_candidate(), - self.keys())) + self.fallbacks = list(map(lambda x: self[x], + filter(lambda x: self[x].is_candidate(), + self.keys()))) # sort fallbacks by their consensus weight to advertised bandwidth factor, # lowest to highest @@ -2084,8 +2084,8 @@ class CandidateList(dict): # now we have at least max_count successful candidates, # or we've tried them all original_count = len(self.fallbacks) - self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(), - self.fallbacks) + self.fallbacks = list(filter(lambda x: x.get_fallback_download_consensus(), + self.fallbacks)) # some of these failed the check, others skipped the check, # if we already had enough successful downloads failed_count = original_count - len(self.fallbacks) @@ -2233,15 +2233,16 @@ class CandidateList(dict): # return a list of fallbacks which are on the IPv4 ORPort port def fallbacks_on_ipv4_orport(self, port): - return filter(lambda x: x.orport == port, self.fallbacks) + return list(filter(lambda x: x.orport == port, self.fallbacks)) # return a list of fallbacks which are on the IPv6 ORPort port def fallbacks_on_ipv6_orport(self, port): - return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6()) + return list(filter(lambda x: x.ipv6orport == port, + self.fallbacks_with_ipv6())) # return a list of fallbacks which are on the DirPort port def fallbacks_on_dirport(self, port): - return filter(lambda x: x.dirport == port, self.fallbacks) + return list(filter(lambda x: x.dirport == port, self.fallbacks)) # log a message about the proportion of fallbacks on IPv4 ORPort port # and return that count @@ -2302,7 +2303,7 @@ class CandidateList(dict): # return a list of fallbacks which cache extra-info documents def fallbacks_with_extra_info_cache(self): - return filter(lambda x: x._extra_info_cache, self.fallbacks) + return list(filter(lambda x: x._extra_info_cache, self.fallbacks)) # log a message about the proportion of fallbacks that cache extra-info docs def describe_fallback_extra_info_caches(self): @@ -2314,7 +2315,7 @@ class CandidateList(dict): # return a list of fallbacks which have the Exit flag def fallbacks_with_exit(self): - return filter(lambda x: x.is_exit(), self.fallbacks) + return list(filter(lambda x: x.is_exit(), self.fallbacks)) # log a message about the proportion of fallbacks with an Exit flag def describe_fallback_exit_flag(self): @@ -2326,7 +2327,7 @@ class CandidateList(dict): # return a list of fallbacks which have an IPv6 address def fallbacks_with_ipv6(self): - return filter(lambda x: x.has_ipv6(), self.fallbacks) + return list(filter(lambda x: x.has_ipv6(), self.fallbacks)) # log a message about the proportion of fallbacks on IPv6 def describe_fallback_ip_family(self):

1 0

[fallback-scripts/master] update: py3: Use new py3 module and function names in the code
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 821404cd02144c55f50ed55df7daa350260d23c7 Author: teor <teor(a)torproject.org> Date: Mon Dec 2 15:31:52 2019 +1000 update: py3: Use new py3 module and function names in the code Part of 28863. --- updateFallbackDirs.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index 9a8e400..57888a9 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -553,7 +553,7 @@ def parse_fallback_file(file_name): def load_possibly_compressed_response_json(response): if response.info().get('Content-Encoding') == 'gzip': - buf = StringIO.StringIO( response.read() ) + buf = six.BytesIO( response.read() ) f = gzip.GzipFile(fileobj=buf) return json.load(f) else: @@ -597,7 +597,7 @@ def onionoo_fetch(what, **kwargs): # then use first_seen to get a stable order. # The order is important when we're limiting the number of relays returned. params['order'] = '-consensus_weight,first_seen' - url = ONIONOO + what + '?' + urllib.urlencode(params) + url = ONIONOO + what + '?' + urllib.parse.urlencode(params) # Unfortunately, the URL is too long for some OS filenames, # but we still don't want to get files from different URLs mixed up @@ -619,7 +619,7 @@ def onionoo_fetch(what, **kwargs): # no need to compare as long as you trust SHA-1 write_to_file(url, full_url_file_name, MAX_FULL_URL_LENGTH) - request = urllib2.Request(url) + request = urllib.request.Request(url) request.add_header('Accept-encoding', 'gzip') # load the last modified date from the file, if it exists @@ -641,9 +641,9 @@ def onionoo_fetch(what, **kwargs): # Make the Onionoo request response_code = 0 try: - response = urllib2.urlopen(request) + response = urllib.request.urlopen(request) response_code = response.getcode() - except urllib2.HTTPError as error: + except urllib.error.HTTPError as error: response_code = error.code if response_code == 304: # not modified pass @@ -890,7 +890,7 @@ class Candidate(object): # checks both recommended versions and bug #20499 / #20509 # # if the relay doesn't have a recommended version field, exclude the relay - if not self._data.has_key('recommended_version'): + if 'recommended_version' not in self._data: log_excluded('%s not a candidate: no recommended_version field', self._fpr) return False @@ -898,7 +898,7 @@ class Candidate(object): log_excluded('%s not a candidate: version not recommended', self._fpr) return False # if the relay doesn't have version field, exclude the relay - if not self._data.has_key('version'): + if 'version' not in self._data: log_excluded('%s not a candidate: no version field', self._fpr) return False if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS: @@ -1080,7 +1080,7 @@ class Candidate(object): log_excluded('%s not a candidate: guard avg too low (%lf)', self._fpr, self._guard) return False - if (not self._data.has_key('consensus_weight') + if ('consensus_weight' not in self._data or self._data['consensus_weight'] < 1): log_excluded('%s not a candidate: consensus weight invalid', self._fpr) return False @@ -1210,7 +1210,7 @@ class Candidate(object): '%s:%d?', self._fpr, self.dirip, int(entry['orport']), self.dirip, self.orport) return False - if entry.has_key('ipv6') and self.has_ipv6(): + if 'ipv6' in entry and self.has_ipv6(): # if both entry and fallback have an ipv6 address, compare them if not self.ipv6_and_orport_matches(entry['ipv6_addr'], entry['ipv6_orport'], @@ -1221,11 +1221,11 @@ class Candidate(object): return False # if the fallback has an IPv6 address but the whitelist entry # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and not self.has_ipv6(): + elif 'ipv6' in entry and not self.has_ipv6(): logging.warning('%s excluded: has it lost its former IPv6 address %s?', self._fpr, entry['ipv6']) return False - elif not entry.has_key('ipv6') and self.has_ipv6(): + elif 'ipv6' not in entry and self.has_ipv6(): logging.warning('%s excluded: has it gained an IPv6 address %s:%d?', self._fpr, self.ipv6addr, self.ipv6orport) return False @@ -1246,7 +1246,7 @@ class Candidate(object): return True if self.ipv4_addr_matches(entry['ipv4'], exact=False): return True - if entry.has_key('ipv6') and self.has_ipv6(): + if 'ipv6' in entry and self.has_ipv6(): # if both entry and fallback have an ipv6 address, compare them if self.ipv6_addr_matches(entry['ipv6_addr'], exact=False): return True @@ -1496,7 +1496,7 @@ class Candidate(object): if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS: return True # if we are performing checks, but haven't done one, return False - if not self._data.has_key('download_check'): + if 'download_check' not in self._data: return False return self._data['download_check'] @@ -2012,7 +2012,7 @@ class CandidateList(dict): max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS remaining_list = fingerprint_list desc_list = [] - for _ in xrange(max_retries): + for _ in six.moves.range(max_retries): if len(remaining_list) == 0: break new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS])

1 0

[fallback-scripts/master] update: py3: Deal with string encoding and type issues
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit a4fbed37e6bca92ea19a2b10fddc15b5c007c3aa Author: teor <teor(a)torproject.org> Date: Mon Dec 2 15:40:24 2019 +1000 update: py3: Deal with string encoding and type issues Part of 28863. --- updateFallbackDirs.py | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index 57888a9..e97a501 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -69,8 +69,8 @@ HAVE_IPADDRESS = False try: # python 3 builtin, or install package py2-ipaddress # there are several ipaddress implementations for python 2 - # with slightly different semantics with str typed text - # fortunately, all our IP addresses are in unicode + # with slightly different semantics with bytes + # to avoid these issues, we make sure our IP addresses are in six.text_type import ipaddress HAVE_IPADDRESS = True except ImportError: @@ -601,7 +601,7 @@ def onionoo_fetch(what, **kwargs): # Unfortunately, the URL is too long for some OS filenames, # but we still don't want to get files from different URLs mixed up - base_file_name = what + '-' + hashlib.sha1(url).hexdigest() + base_file_name = what + '-' + hashlib.sha1(url.encode('ascii')).hexdigest() full_url_file_name = base_file_name + '.full_url' MAX_FULL_URL_LENGTH = 1024 @@ -712,8 +712,6 @@ class Candidate(object): 'consensus_weight', 'or_addresses', 'dir_address']: if not f in details: raise Exception("Document has no %s field."%(f,)) - if not 'contact' in details: - details['contact'] = None if not 'flags' in details or details['flags'] is None: details['flags'] = [] if (not 'advertised_bandwidth' in details @@ -724,10 +722,19 @@ class Candidate(object): if (not 'effective_family' in details or details['effective_family'] is None): details['effective_family'] = [] - if not 'platform' in details: - details['platform'] = None details['last_changed_address_or_port'] = parse_ts( details['last_changed_address_or_port']) + + # Handle fields that can have arbitrary bytes, but should be UTF-8 + if not 'contact' in details: + details['contact'] = None + else: + details['contact'] = six.ensure_text(details['contact'], errors='replace') + if not 'platform' in details: + details['platform'] = None + else: + details['platform'] = six.ensure_text(details['platform'], errors='replace') + self._data = details self._stable_sort_or_addresses() @@ -761,7 +768,7 @@ class Candidate(object): # is_valid_ipv[46]_address by gsathya, karsten, 2013 @staticmethod def is_valid_ipv4_address(address): - if not isinstance(address, (str, unicode)): + if not isinstance(address, six.string_types): return False # check if there are four period separated values @@ -779,7 +786,7 @@ class Candidate(object): @staticmethod def is_valid_ipv6_address(address): - if not isinstance(address, (str, unicode)): + if not isinstance(address, six.string_types): return False # remove brackets @@ -814,6 +821,7 @@ class Candidate(object): def _split_dirport(self): # Split the dir_address into dirip and dirport + self._data['dir_address'] = six.ensure_text(self._data['dir_address']) (self.dirip, _dirport) = self._data['dir_address'].split(':', 2) self.dirport = int(_dirport) @@ -827,6 +835,7 @@ class Candidate(object): for i in self._data['or_addresses']: if i != self._data['or_addresses'][0]: logging.debug('Secondary IPv4 Address Used for %s: %s'%(self._fpr, i)) + i = six.ensure_text(i) (ipaddr, port) = i.rsplit(':', 1) if (ipaddr == self.dirip) and Candidate.is_valid_ipv4_address(ipaddr): self.orport = int(port) @@ -841,17 +850,21 @@ class Candidate(object): self.ipv6orport = None # Choose the first IPv6 address that uses the same port as the ORPort for i in self._data['or_addresses']: + i = six.ensure_text(i) (ipaddr, port) = i.rsplit(':', 1) + port = int(port) if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr): self.ipv6addr = ipaddr - self.ipv6orport = int(port) + self.ipv6orport = port return # Choose the first IPv6 address in the list for i in self._data['or_addresses']: + i = six.ensure_text(i) (ipaddr, port) = i.rsplit(':', 1) + port = int(port) if Candidate.is_valid_ipv6_address(ipaddr): self.ipv6addr = ipaddr - self.ipv6orport = int(port) + self.ipv6orport = port return def _compute_version(self): @@ -1306,16 +1319,16 @@ class Candidate(object): # strip leading and trailing brackets from an IPv6 address # safe to use on non-bracketed IPv6 and on IPv4 addresses - # also convert to unicode, and make None appear as '' + # also make None appear as '' @staticmethod def strip_ipv6_brackets(ip): if ip is None: - return unicode('') + return '' if len(ip) < 2: - return unicode(ip) + return ip if ip[0] == '[' and ip[-1] == ']': - return unicode(ip[1:-1]) - return unicode(ip) + return ip[1:-1] + return ip # are ip_a and ip_b in the same netblock? # mask_bits is the size of the netblock

1 0

[fallback-scripts/master] update: py3: Use floor division when needed
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit ed453731fcd801826ce75f31727ed7bc78b4f640 Author: teor <teor(a)torproject.org> Date: Mon Dec 2 16:06:08 2019 +1000 update: py3: Use floor division when needed Part of 28863. --- updateFallbackDirs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index a0c8ad6..d557b7c 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -1546,7 +1546,7 @@ class Candidate(object): bandwidth = self._data['measured_bandwidth'] weight = self._data['consensus_weight'] s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%( - bandwidth/(1024.0*1024.0), + bandwidth // (1024.0*1024.0), weight) s += '\n' if self._data['contact'] is not None: @@ -1856,7 +1856,7 @@ class CandidateList(dict): # use the low-median when there are an evan number of fallbacks, # for consistency with the bandwidth authorities if len(self.fallbacks) > 0: - median_position = (len(self.fallbacks) - 1) / 2 + median_position = (len(self.fallbacks) - 1) // 2 if not require_advertised_bandwidth: return self.fallbacks[median_position] # if we need advertised_bandwidth but this relay doesn't have it, @@ -2022,7 +2022,7 @@ class CandidateList(dict): # descriptors # # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list - max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS + max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) // MAX_FINGERPRINTS remaining_list = fingerprint_list desc_list = [] for _ in six.moves.range(max_retries):

1 0

[fallback-scripts/master] Travis: Add spaces and comments for clarity
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 576a943e94a5bf3e6c786d8319aa773abfaf0560 Author: teor <teor(a)torproject.org> Date: Wed Dec 18 12:23:11 2019 +1000 Travis: Add spaces and comments for clarity --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index f44d253..5927199 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,20 +56,25 @@ matrix: ## Pre-installed in Travis bionic: ## https://docs.travis-ci.com/user/reference/bionic/#python-support + ## End of Life: 1 January 2020 ## https://www.python.org/dev/peps/pep-0373/#update - python: "2.7" + ## End of Life: December 2021 ## https://www.python.org/dev/peps/pep-0494/#lifespan - python: "3.6" + ## End of Life: June 2023 ## https://www.python.org/dev/peps/pep-0537/#lifespan - python: "3.7" ## Extra Installs + ## End of Life: October 2024 ## https://www.python.org/dev/peps/pep-0569/#lifespan - python: "3.8" + ## Python 3.9 ## Travis Dev Package: ???? ## (Add 3.9-dev) @@ -78,16 +83,20 @@ matrix: ## End of Life: October 2025 ## https://www.python.org/dev/peps/pep-0596/#lifespan #- python: "3.9-dev" + - python: "nightly" ## PyPy versions ## PyPy isn't packaged for Travis Bionic yet + + ## PyPy 2 ## End of Life: "forever" ## http://doc.pypy.org/en/latest/faq.html#how-long-will-pypy-support-python2 ## But stem or fallback-scripts may decide not to support python 2 ## after 1 Jan 2020. - python: "pypy" dist: xenial + ## PyPy does not have documented end of life dates - python: "pypy3" dist: xenial

1 0

[fallback-scripts/master] scripts: Use __future__ imports to adopt 3.0 behaviour
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit d55e956d988b6efe567c447c948c20349ff559aa Author: teor <teor(a)torproject.org> Date: Thu Dec 12 15:36:17 2019 +1000 scripts: Use __future__ imports to adopt 3.0 behaviour Part of 28863. --- generateFallbackDirLine.py | 5 +++++ lookupFallbackDirContact.py | 5 +++++ updateFallbackDirs.py | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/generateFallbackDirLine.py b/generateFallbackDirLine.py index 768c98d..1766674 100755 --- a/generateFallbackDirLine.py +++ b/generateFallbackDirLine.py @@ -6,6 +6,11 @@ # Usage: # generateFallbackDirLine.py fingerprint ... +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import sys import stem.descriptor.remote as remote diff --git a/lookupFallbackDirContact.py b/lookupFallbackDirContact.py index 8db4714..d656956 100755 --- a/lookupFallbackDirContact.py +++ b/lookupFallbackDirContact.py @@ -6,6 +6,11 @@ # Usage: # lookupFallbackDirContact.py fingerprint ... +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import sys import stem.descriptor.remote as remote diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index d557b7c..86343d4 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -35,6 +35,11 @@ # https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.… # Modifications by teor, 2015 +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + ## Imports: version-independent import copy

1 0

[fallback-scripts/master] Travis: Use our standard set of python versions
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 06ada8031ad91e86f9324c321098a87ee1d854c6 Author: teor <teor(a)torproject.org> Date: Mon Dec 2 17:25:02 2019 +1000 Travis: Use our standard set of python versions Part of 28863. --- .travis.yml | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2d0bcf4..f44d253 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,11 +24,15 @@ matrix: # include creates Linux, python 2.7 builds by default # the key(s) in each item override these defaults include: - ## macOS on Travis is tricky: + ## Test all supported python releases + + ## python in macOS on Travis is tricky: ## - We use the default python version on macOS, which is currently 2.7. ## (But we don't show the version, because Travis might change it ## without us noticing.) ## - We use language: c, because language: python fails on Travis macOS. + + ## We need to install python@2 in homebrew, to get pip[2] - os: osx language: c python: @@ -36,40 +40,55 @@ matrix: ## (macOS only) homebrew: packages: - ## macOS needs pip(2) from homebrew python + ## macOS needs pip[2] from homebrew python - python@2 - shellcheck ## See ticket #30928 for more information. This key should be ## removed at some point to speed up builds. update: true + + ## Pre-installed in macOS - os: osx language: c python: - ## python 3 is already installed, under these names + ## python 3 is installed under these names env: PYTHON="python3" PIP="pip3" - ## Test all supported python releases - ## Pre-installed in Travis Bionic: + ## Pre-installed in Travis bionic: ## https://docs.travis-ci.com/user/reference/bionic/#python-support ## End of Life: 1 January 2020 ## https://www.python.org/dev/peps/pep-0373/#update - python: "2.7" - - ### TODO: the fallback scripts don't work with python 3 yet ### - ### See chutney's .travis.yml for the latest python version list ### + ## End of Life: December 2021 + ## https://www.python.org/dev/peps/pep-0494/#lifespan + - python: "3.6" + ## End of Life: June 2023 + ## https://www.python.org/dev/peps/pep-0537/#lifespan + - python: "3.7" + + ## Extra Installs + ## End of Life: October 2024 + ## https://www.python.org/dev/peps/pep-0569/#lifespan + - python: "3.8" + ## Python 3.9 + ## Travis Dev Package: ???? + ## (Add 3.9-dev) + ## Stable: 10 October 2020 + ## (Switch from 3.9-dev to 3.9, and check for {3.10,4.0}-dev) + ## End of Life: October 2025 + ## https://www.python.org/dev/peps/pep-0596/#lifespan + #- python: "3.9-dev" + - python: "nightly" ## PyPy versions + ## PyPy isn't packaged for Travis Bionic yet ## End of Life: "forever" ## http://doc.pypy.org/en/latest/faq.html#how-long-will-pypy-support-python2 ## But stem or fallback-scripts may decide not to support python 2 ## after 1 Jan 2020. - python: "pypy" dist: xenial - - ### TODO: the fallback scripts don't work with python 3 yet ### - ### See chutney's .travis.yml for the latest python version list ### - - python: "3.6" - - python: "nightly" + ## PyPy does not have documented end of life dates - python: "pypy3" dist: xenial @@ -83,11 +102,7 @@ matrix: ## These builds fail in Travis at the moment allow_failures: - - python: "3.6" - - python: "nightly" - - python: "pypy3" - ## (macOS only) - - env: PYTHON="python3" PIP="pip3" + #- python: "nightly" ## (Linux only) Use a recent Linux image (Ubuntu Bionic) dist: bionic

1 0

[fallback-scripts/master] Merge remote-tracking branch 'tor-github/pr/9'
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 77f850359ac2151c40da9dc51f40d390e83ebd09 Merge: 0a715b5 576a943 Author: teor <teor(a)torproject.org> Date: Mon Jan 20 15:11:13 2020 +1000 Merge remote-tracking branch 'tor-github/pr/9' Merge python 3 fixes with offer-list fixes. .travis.yml | 78 ++++++++++++------ generateFallbackDirLine.py | 55 +++++++++---- lookupFallbackDirContact.py | 43 +++++++--- requirements.txt | 1 + test.sh | 39 +++++++++ updateFallbackDirs.py | 189 +++++++++++++++++++++++++------------------- 6 files changed, 272 insertions(+), 133 deletions(-) diff --cc updateFallbackDirs.py index 25aa4c4,86343d4..c2e63cf --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@@ -1211,9 -1237,9 +1237,9 @@@ class Candidate(object) 'to %s:%d?', self._fpr, entry['ipv6'], self.ipv6addr, self.ipv6orport) return False - # if the fallback has an IPv6 address but the whitelist entry - # doesn't, or vice versa, the whitelist entry doesn't match + # if the fallback has an IPv6 address but the offer list entry + # doesn't, or vice versa, the offer list entry doesn't match - elif entry.has_key('ipv6') and not self.has_ipv6(): + elif 'ipv6' in entry and not self.has_ipv6(): logging.warning('%s excluded: has it lost its former IPv6 address %s?', self._fpr, entry['ipv6']) return False @@@ -2421,22 -2446,19 +2448,22 @@@ def list_fallbacks(offer_list, exact=Fa """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays, passing exact to apply_filter_lists(). """ - print "/* type=fallback */" - print ("/* version={} */" - .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) + print("/* type=fallback */") + print("/* version={} */" + .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) now = datetime.datetime.utcnow() timestamp = now.strftime('%Y%m%d%H%M%S') - print("/* timestamp={} */" - .format(cleanse_c_multiline_comment(timestamp))) - if whitelist['check_existing']: + print ("/* timestamp={} */" + .format(cleanse_c_multiline_comment(timestamp))) + if offer_list['check_existing']: - print "/* source=fallback */" + print("/* source=fallback */") else: - print("/* source=whitelist */") + # TODO: when we implement #24839, change this to descriptor,offer-list + # We might want add each key based on the size of the offer list, and + # the number of descriptors with offer-fallback-dir lines. - print "/* source=offer-list */" ++ print("/* source=offer-list */") # end the header with a separator, to make it easier for parsers - print SECTION_SEPARATOR_COMMENT + print(SECTION_SEPARATOR_COMMENT) logging.warning('Downloading and parsing Onionoo data. ' + 'This may take some time.') @@@ -2461,15 -2483,15 +2488,15 @@@ candidates.compute_fallbacks() prefilter_fallbacks = copy.copy(candidates.fallbacks) - # filter with the whitelist + # filter with the offer list # if a relay has changed IPv4 address or ports recently, it will be excluded # as ineligible before we call apply_filter_lists, and so there will be no - # warning that the details have changed from those in the whitelist. + # warning that the details have changed from those in the offer list. # instead, there will be an info-level log during the eligibility check. initial_count = len(candidates.fallbacks) - excluded_count = candidates.apply_filter_lists(whitelist, exact=exact) + excluded_count = candidates.apply_filter_lists(offer_list, exact=exact) - print candidates.summarise_filters(initial_count, excluded_count, - offer_list['check_existing']) + print(candidates.summarise_filters(initial_count, excluded_count, - whitelist['check_existing'])) ++ offer_list['check_existing'])) eligible_count = len(candidates.fallbacks) # calculate the measured bandwidth of each relay, @@@ -2520,12 -2542,12 +2547,12 @@@ # output C comments summarising the fallback selection process if len(candidates.fallbacks) > 0: - print candidates.summarise_fallbacks(eligible_count, operator_count, + print(candidates.summarise_fallbacks(eligible_count, operator_count, failed_count, guard_count, target_count, - offer_list['check_existing']) - whitelist['check_existing'])) ++ offer_list['check_existing'])) else: - print '/* No Fallbacks met criteria */' + print('/* No Fallbacks met criteria */') # output C comments specifying the Onionoo data used to create the list for s in fetch_source_list():

1 0

[fallback-scripts/master] scripts: Rewrite generate and lookup to be more robust
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 9f8689b8982bac8820cb9f5225992866fed0d215 Author: teor <teor(a)torproject.org> Date: Mon Dec 2 18:13:22 2019 +1000 scripts: Rewrite generate and lookup to be more robust Reduce the number of connections in generate. Simplify and remove compatibility code. Increase retries, and add a timeout. Part of 28863. --- generateFallbackDirLine.py | 61 ++++++++++++++++++++++++--------------------- lookupFallbackDirContact.py | 35 ++++++++++++++++++-------- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/generateFallbackDirLine.py b/generateFallbackDirLine.py index f5e7424..768c98d 100755 --- a/generateFallbackDirLine.py +++ b/generateFallbackDirLine.py @@ -8,44 +8,47 @@ import sys -from six.moves import urllib - -import stem -import stem.descriptor.remote -import stem.util.tor_tools - -## Stem version compatibility -try: - stem_DownloadFailed = stem.DownloadFailed -except AttributeError: - class DummyException(BaseException): - pass - stem_DownloadFailed = DummyException +import stem.descriptor.remote as remote +import stem.util.tor_tools as tor_tools if len(sys.argv) <= 1: print('Usage: %s fingerprint ...' % sys.argv[0]) sys.exit(1) -for fingerprint in sys.argv[1:]: - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): +input_list = sys.argv[1:] + +for fingerprint in input_list: + if not tor_tools.is_valid_fingerprint(fingerprint): print("'%s' isn't a valid relay fingerprint" % fingerprint) sys.exit(1) - try: - desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0] - except urllib.error.HTTPError as exc: - if exc.code == 404: - print('# %s not found in recent descriptors' % fingerprint) - continue - else: - raise - except stem_DownloadFailed as exc: - print('# %s not found by stem' % fingerprint) +found_list = [] +desc_query = remote.get_server_descriptors(input_list, + retries=3, + timeout=30) +for desc in desc_query.run(): + assert desc.fingerprint in input_list + # Skip duplicates on retries + if desc.fingerprint in found_list: continue + found_list.append(desc.fingerprint) if not desc.dir_port: - print("# %s needs a DirPort" % fingerprint) + print("# %s needs a DirPort" % desc.fingerprint) else: - ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6] - ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else '' - print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname)) + ipv6_addresses = [(address, port) + for address, port, is_ipv6 in desc.or_addresses + if is_ipv6] + ipv6_field = (' ipv6=[%s]:%s' % ipv6_addresses[0] + if ipv6_addresses + else '') + print('%s:%s orport=%s id=%s%s # %s' % (desc.address, + desc.dir_port, + desc.or_port, + desc.fingerprint, + ipv6_field, + desc.nickname)) + +for fingerprint in input_list: + if fingerprint not in found_list: + print("# {} not found in current descriptors".format(fingerprint)) diff --git a/lookupFallbackDirContact.py b/lookupFallbackDirContact.py index 07aeae7..8db4714 100755 --- a/lookupFallbackDirContact.py +++ b/lookupFallbackDirContact.py @@ -9,25 +9,38 @@ import sys import stem.descriptor.remote as remote +import stem.util.tor_tools as tor_tools if len(sys.argv) <= 1: print("Usage: {} fingerprint ...".format(sys.argv[0])) sys.exit(-1) +input_list = sys.argv[1:] + +for fingerprint in input_list: + if not tor_tools.is_valid_fingerprint(fingerprint): + print("'%s' isn't a valid relay fingerprint" % fingerprint) + sys.exit(1) + +found_list = [] # we need descriptors, because the consensus does not have contact infos -descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run() +desc_query = remote.get_server_descriptors(input_list, + retries=3, + timeout=30) +for desc in desc_query.run(): + assert desc.fingerprint in input_list + # Skip duplicates on retries + if desc.fingerprint in found_list: + continue + found_list.append(desc.fingerprint) -descriptor_list_fingerprints = [] -for d in descriptor_list: - assert d.fingerprint in sys.argv[1:] - descriptor_list_fingerprints.append(d.fingerprint) - if d.contact: + if desc.contact: # Most ContactInfos should be UTF-8 - contact = d.contact.decode(encoding="utf-8", errors="replace") + contact = desc.contact.decode(encoding="utf-8", errors="replace") else: contact = "(no contact)" - print("{} {}".format(d.fingerprint, contact)) + print("{} {}".format(desc.fingerprint, contact)) -for fingerprint in sys.argv[1:]: - if fingerprint not in descriptor_list_fingerprints: - print("{} # not found in current descriptors".format(fingerprint)) +for fingerprint in input_list: + if fingerprint not in found_list: + print("{} (descriptor not found)".format(fingerprint))

1 0

[fallback-scripts/master] update: Use a package name from Debian when listing dependencies
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit ca9816572e9af0d2a1822eea888cccf0ee913bb5 Author: teor <teor(a)torproject.org> Date: Mon Dec 2 13:54:05 2019 +1000 update: Use a package name from Debian when listing dependencies Comment-only change. Part of 28863. --- updateFallbackDirs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index 02f97e3..77e9bbb 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -23,7 +23,7 @@ # If this is not possible, please disable: # PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS # -# Needs dateutil, stem, and potentially other python packages. +# Needs [python-]dateutil, stem, and potentially other python packages. # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) # for netblock analysis. #

1 0