commit 9f8689b8982bac8820cb9f5225992866fed0d215 Author: teor <teor@torproject.org> Date: Mon Dec 2 18:13:22 2019 +1000 scripts: Rewrite generate and lookup to be more robust Reduce the number of connections in generate. Simplify and remove compatibility code. Increase retries, and add a timeout. Part of 28863. --- generateFallbackDirLine.py | 61 ++++++++++++++++++++++++--------------------- lookupFallbackDirContact.py | 35 ++++++++++++++++++-------- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/generateFallbackDirLine.py b/generateFallbackDirLine.py index f5e7424..768c98d 100755 --- a/generateFallbackDirLine.py +++ b/generateFallbackDirLine.py @@ -8,44 +8,47 @@ import sys -from six.moves import urllib - -import stem -import stem.descriptor.remote -import stem.util.tor_tools - -## Stem version compatibility -try: - stem_DownloadFailed = stem.DownloadFailed -except AttributeError: - class DummyException(BaseException): - pass - stem_DownloadFailed = DummyException +import stem.descriptor.remote as remote +import stem.util.tor_tools as tor_tools if len(sys.argv) <= 1: print('Usage: %s fingerprint ...' % sys.argv[0]) sys.exit(1) -for fingerprint in sys.argv[1:]: - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): +input_list = sys.argv[1:] + +for fingerprint in input_list: + if not tor_tools.is_valid_fingerprint(fingerprint): print("'%s' isn't a valid relay fingerprint" % fingerprint) sys.exit(1) - try: - desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0] - except urllib.error.HTTPError as exc: - if exc.code == 404: - print('# %s not found in recent descriptors' % fingerprint) - continue - else: - raise - except stem_DownloadFailed as exc: - print('# %s not found by stem' % fingerprint) +found_list = [] +desc_query = remote.get_server_descriptors(input_list, + retries=3, + timeout=30) +for desc in desc_query.run(): + assert desc.fingerprint in input_list + # Skip duplicates on retries + if desc.fingerprint in found_list: continue + found_list.append(desc.fingerprint) if not desc.dir_port: - print("# %s needs a DirPort" % fingerprint) + print("# %s needs a DirPort" % desc.fingerprint) else: - ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6] - ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else '' - print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname)) + ipv6_addresses = [(address, port) + for address, port, is_ipv6 in desc.or_addresses + if is_ipv6] + ipv6_field = (' ipv6=[%s]:%s' % ipv6_addresses[0] + if ipv6_addresses + else '') + print('%s:%s orport=%s id=%s%s # %s' % (desc.address, + desc.dir_port, + desc.or_port, + desc.fingerprint, + ipv6_field, + desc.nickname)) + +for fingerprint in input_list: + if fingerprint not in found_list: + print("# {} not found in current descriptors".format(fingerprint)) diff --git a/lookupFallbackDirContact.py b/lookupFallbackDirContact.py index 07aeae7..8db4714 100755 --- a/lookupFallbackDirContact.py +++ b/lookupFallbackDirContact.py @@ -9,25 +9,38 @@ import sys import stem.descriptor.remote as remote +import stem.util.tor_tools as tor_tools if len(sys.argv) <= 1: print("Usage: {} fingerprint ...".format(sys.argv[0])) sys.exit(-1) +input_list = sys.argv[1:] + +for fingerprint in input_list: + if not tor_tools.is_valid_fingerprint(fingerprint): + print("'%s' isn't a valid relay fingerprint" % fingerprint) + sys.exit(1) + +found_list = [] # we need descriptors, because the consensus does not have contact infos -descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run() +desc_query = remote.get_server_descriptors(input_list, + retries=3, + timeout=30) +for desc in desc_query.run(): + assert desc.fingerprint in input_list + # Skip duplicates on retries + if desc.fingerprint in found_list: + continue + found_list.append(desc.fingerprint) -descriptor_list_fingerprints = [] -for d in descriptor_list: - assert d.fingerprint in sys.argv[1:] - descriptor_list_fingerprints.append(d.fingerprint) - if d.contact: + if desc.contact: # Most ContactInfos should be UTF-8 - contact = d.contact.decode(encoding="utf-8", errors="replace") + contact = desc.contact.decode(encoding="utf-8", errors="replace") else: contact = "(no contact)" - print("{} {}".format(d.fingerprint, contact)) + print("{} {}".format(desc.fingerprint, contact)) -for fingerprint in sys.argv[1:]: - if fingerprint not in descriptor_list_fingerprints: - print("{} # not found in current descriptors".format(fingerprint)) +for fingerprint in input_list: + if fingerprint not in found_list: + print("{} (descriptor not found)".format(fingerprint))