February 2016 - tor-commits - lists.torproject.org

[stem/master] Script for caching fallback directories
by atagar＠torproject.org 22 Feb '16

22 Feb '16

commit 9e3f6a2930893bcf4cd69a7e18c00efb2480ff10 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 21 18:19:16 2016 -0800 Script for caching fallback directories Oops! Forgot to add a couple new files to support fallback directories. --- cache_fallback_directories.py | 64 ++++++++++++++++++++++++ stem/descriptor/fallback_directories.cfg | 86 ++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) diff --git a/cache_fallback_directories.py b/cache_fallback_directories.py new file mode 100755 index 0000000..d6a58a9 --- /dev/null +++ b/cache_fallback_directories.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# Copyright 2016, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Caches tor's latest fallback directories. +""" + +import re +import sys + +import stem.descriptor.remote +import stem.util.conf +import stem.util.system + +try: + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib +except ImportError: + import urllib2 as urllib + +GITWEB_MAN_LOG = 'https://gitweb.torproject.org/tor.git/log/src/or/fallback_dirs.inc' +FALLBACK_DIR_LINK = "href='/tor.git/commit/src/or/fallback_dirs.inc\?id=([^']*)'" + +if __name__ == '__main__': + try: + fallback_dir_page = urllib.urlopen(GITWEB_MAN_LOG).read() + fallback_dir_commit = re.search(FALLBACK_DIR_LINK, fallback_dir_page).group(1) + except: + print("Unable to determine the latest commit to edit tor's fallback directories: %s" % sys.exc_info()[1]) + sys.exit(1) + + try: + stem_commit = stem.util.system.call('git rev-parse HEAD')[0] + except IOError as exc: + print("Unable to determine stem's current commit: %s" % exc) + sys.exit(1) + + print('Latest tor commit editing fallback directories: %s' % fallback_dir_commit) + print('Current stem commit: %s' % stem_commit) + print('') + + cached_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() + latest_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() + + if cached_fallback_directories == latest_fallback_directories: + print('Fallback directories are already up to date, nothing to do.') + sys.exit(0) + + print('Differences detected...\n') + print(stem.descriptor.remote._fallback_directory_differences(cached_fallback_directories, latest_fallback_directories)) + + conf = stem.util.conf.Config() + conf.set('tor_commit', fallback_dir_commit) + conf.set('stem_commit', stem_commit) + + for directory in latest_fallback_directories.values(): + nickname = directory.nickname + conf.set('%s.address' % nickname, directory.address) + conf.set('%s.or_port' % nickname, str(directory.or_port)) + conf.set('%s.dir_port' % nickname, str(directory.dir_port)) + conf.set('%s.fingerprint' % nickname, directory.fingerprint) + + conf.save(stem.descriptor.remote.CACHE_PATH) diff --git a/stem/descriptor/fallback_directories.cfg b/stem/descriptor/fallback_directories.cfg new file mode 100644 index 0000000..158378d --- /dev/null +++ b/stem/descriptor/fallback_directories.cfg @@ -0,0 +1,86 @@ +tor_commit ab3c86479a516073e0b445846a22b0e24ce8078e +stem_commit 645017a5f8c17b143e90602fdfe6c03a1a994a1c +Doedel22.address 178.254.44.135 +Doedel22.or_port 9001 +Doedel22.dir_port 9030 +Doedel22.fingerprint 8FA37B93397015B2BC5A525C908485260BE9F422 +tornoderdednl.address 178.62.199.226 +tornoderdednl.or_port 443 +tornoderdednl.dir_port 80 +tornoderdednl.fingerprint CBEFF7BA4A4062045133C053F2D70524D8BBE5BE +Logforme.address 84.219.173.60 +Logforme.or_port 443 +Logforme.dir_port 9030 +Logforme.fingerprint 855BC2DABE24C861CD887DB9B2E950424B49FC34 +Doedel21.address 178.254.44.135 +Doedel21.or_port 443 +Doedel21.dir_port 80 +Doedel21.fingerprint AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2 +BabylonNetwork02.address 149.202.98.161 +BabylonNetwork02.or_port 443 +BabylonNetwork02.dir_port 80 +BabylonNetwork02.fingerprint 54660C671B47E6986B465B80444414BD19E5A34B +fluxe3.address 78.47.18.110 +fluxe3.or_port 80 +fluxe3.dir_port 443 +fluxe3.fingerprint F8D27B163B9247B232A2EEE68DD8B698695C28DE +kili.address 178.62.173.203 +kili.or_port 9001 +kili.dir_port 9030 +kili.fingerprint DD85503F2D1F52EF9EAD621E942298F46CD2FC10 +wagner.address 5.175.233.86 +wagner.or_port 443 +wagner.dir_port 80 +wagner.fingerprint 5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33 +fluxe4.address 131.188.40.188 +fluxe4.or_port 80 +fluxe4.dir_port 443 +fluxe4.fingerprint EBE718E1A49EE229071702964F8DB1F318075FF8 +kitten2.address 62.210.124.124 +kitten2.or_port 9101 +kitten2.dir_port 9130 +kitten2.fingerprint 2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E +coby.address 51.255.33.237 +coby.or_port 9001 +coby.dir_port 9091 +coby.fingerprint A360C21FA87FFA2046D92C17086A6B47E5C68109 +GrmmlLitavis.address 5.39.88.19 +GrmmlLitavis.or_port 9001 +GrmmlLitavis.dir_port 9030 +GrmmlLitavis.fingerprint 7CB8C31432A796731EA7B6BF4025548DFEB25E0C +Doedel24.address 178.254.20.134 +Doedel24.or_port 9001 +Doedel24.dir_port 9030 +Doedel24.fingerprint 2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC +BabylonNetwork03.address 193.111.136.162 +BabylonNetwork03.or_port 443 +BabylonNetwork03.dir_port 80 +BabylonNetwork03.fingerprint C79552275DFCD486B942510EF663ED36ACA1A84B +Unnamed.address 217.12.199.208 +Unnamed.or_port 443 +Unnamed.dir_port 80 +Unnamed.fingerprint DF3AED4322B1824BF5539AE54B2D1B38E080FF05 +eriador.address 85.25.138.93 +eriador.or_port 4029 +eriador.dir_port 9030 +eriador.fingerprint 6DE61A6F72C1E5418A66BFED80DFB63E4C77668F +Doedel26.address 178.254.20.134 +Doedel26.or_port 443 +Doedel26.dir_port 80 +Doedel26.fingerprint 9F5068310818ED7C70B0BC4087AB55CB12CB4377 +PedicaboMundi.address 144.76.14.145 +PedicaboMundi.or_port 143 +PedicaboMundi.dir_port 110 +PedicaboMundi.fingerprint 14419131033443AE6E21DA82B0D307F7CAE42BDB +kitten1.address 62.210.124.124 +kitten1.or_port 9001 +kitten1.dir_port 9030 +kitten1.fingerprint 86E78DD3720C78DA8673182EF96C54B162CD660C +Binnacle.address 108.53.208.157 +Binnacle.or_port 443 +Binnacle.dir_port 80 +Binnacle.fingerprint 4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2 +kitten4.address 212.47.237.95 +kitten4.or_port 9101 +kitten4.dir_port 9130 +kitten4.fingerprint 6FB38EB22E57EF7ED5EF00238F6A48E553735D88

1 0

[stem/master] Fallback directory information
by atagar＠torproject.org 22 Feb '16

22 Feb '16

commit 332ef6e6fe89d9f0d48e9b95acff7de9f3ec647f Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 21 18:04:40 2016 -0800 Fallback directory information Recently tor added fallback directory authorities to help clients bootstrap when the dirauths are unavailable... https://trac.torproject.org/projects/tor/ticket/17158 To help support this effort teor asked for a DocTor check to notify us when the directories become unavailable... https://trac.torproject.org/projects/tor/ticket/18177 Great idea but I'm expanding this. Stem now has a FallbackDirectory class with two methods for getting this information... * FallbackDirectory.from_remote() - Reads the latest fallback_dirs.inc from gitweb, providing the latest fallback directories in tor's master branch. * FallbackDirectory.from_cache() - Provides the latest fallback directories Stem has cached. This is only as up-to-date as the Stem release you're using but is quicker and avoids relying on gitweb. Advantages are... * Stem's descriptor.remote module now puts less load on the directory authorities since it uses fallback directories as well. * Running Stem's integ tests with the ONLINE target includes a test that exercises all the fallback directories, notifying us if any are down. --- cache_manual.py | 4 +- docs/change_log.rst | 1 + run_tests.py | 1 + setup.py | 2 +- stem/descriptor/remote.py | 266 ++++++++++++++++++++++++++++++++++++---- test/integ/descriptor/remote.py | 38 ++++++ test/unit/descriptor/remote.py | 56 +++++++++ 7 files changed, 341 insertions(+), 27 deletions(-) diff --git a/cache_manual.py b/cache_manual.py index aa55dc5..19e73f0 100755 --- a/cache_manual.py +++ b/cache_manual.py @@ -6,7 +6,6 @@ Caches tor's latest manual content. Run this to pick new man page changes. """ -import os import re import sys @@ -19,7 +18,6 @@ try: except ImportError: import urllib2 as urllib -CACHE_PATH = os.path.join(os.path.dirname(__file__), 'stem', 'cached_tor_manual.cfg') GITWEB_MAN_LOG = 'https://gitweb.torproject.org/tor.git/log/doc/tor.1.txt' MAN_LOG_LINK = "href='/tor.git/commit/doc/tor.1.txt\?id=([^']*)'" @@ -53,4 +51,4 @@ if __name__ == '__main__': latest_manual.man_commit = man_commit latest_manual.stem_commit = stem_commit - latest_manual.save(CACHE_PATH) + latest_manual.save(stem.manual.CACHE_PATH) diff --git a/docs/change_log.rst b/docs/change_log.rst index 81bf5a2..c8dc434 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -58,6 +58,7 @@ The following are only available within Stem's `git repository * **Descriptors** + * Fallback directory information, lessing the load of `stem.descriptor.remote <api/descriptor/remote.html>`_ on the directory authorities * Support for ed25519 descriptor fields (:spec:`5a79d67`) * Server descriptor validation fails with 'extra-info-digest line had an invalid value' from additions in proposal 228 (:trac:`16227`) * :class:`~stem.descriptor.server_descriptor.BridgeDescriptor` now has 'ntor_onion_key' like its unsanitized counterparts diff --git a/run_tests.py b/run_tests.py index 3d0f845..2fb93bf 100755 --- a/run_tests.py +++ b/run_tests.py @@ -42,6 +42,7 @@ SRC_PATHS = [os.path.join(STEM_BASE, path) for path in ( 'test', 'run_tests.py', 'cache_manual.py', + 'cache_fallback_directories.py', 'setup.py', 'tor-prompt', os.path.join('docs', 'republish.py'), diff --git a/setup.py b/setup.py index 2a78b9b..2e492d4 100644 --- a/setup.py +++ b/setup.py @@ -17,5 +17,5 @@ distutils.core.setup( packages = ['stem', 'stem.descriptor', 'stem.interpreter', 'stem.response', 'stem.util'], keywords = 'tor onion controller', scripts = ['tor-prompt'], - package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']}, + package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.descriptor': ['fallback_directories.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']}, ) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index d199fb2..dc40e41 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -44,7 +44,11 @@ itself... get_authorities - Provides tor directory information. - DirectoryAuthority - Information about a tor directory authority. + Directory - Relay we can retrieve directory information from + |- DirectoryAuthority - Information about a tor directory authority + +- FallbackDirectory - Directory mirror tor uses when authories are unavailable + |- from_cache - Provides fallback directories cached with Stem. + +- from_remote - Retrieves fallback directories remotely from tor's latest commit. Query - Asynchronous request to download tor descriptors |- start - issues the query if it isn't already running @@ -73,7 +77,9 @@ itself... """ import io +import os import random +import re import sys import threading import time @@ -88,7 +94,7 @@ except ImportError: import stem.descriptor from stem import Flag -from stem.util import log +from stem.util import connection, log, tor_tools # Tor has a limited number of descriptors we can fetch explicitly by their # fingerprint or hashes due to a limit on the url length by squid proxies. @@ -96,6 +102,9 @@ from stem.util import log MAX_FINGERPRINTS = 96 MAX_MICRODESCRIPTOR_HASHES = 92 +GITWEB_FALLBACK_DIR_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' +CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg') + def _guess_descriptor_type(resource): # Attempts to determine the descriptor type based on the resource url. This @@ -340,8 +349,10 @@ class Query(object): """ if use_authority or not self.endpoints: - authority = random.choice(list(filter(lambda auth: auth.v3ident is not None, get_authorities().values()))) - address, dirport = authority.address, authority.dir_port + directories = get_authorities().values() + FallbackDirectory.from_cache().values() + + picked = random.choice(directories) + address, dirport = picked.address, picked.dir_port else: address, dirport = random.choice(self.endpoints) @@ -390,8 +401,8 @@ class DescriptorDownloader(object): def __init__(self, use_mirrors = False, **default_args): self._default_args = default_args - authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values()) - self._endpoints = [(auth.address, auth.dir_port) for auth in authorities] + directories = get_authorities().values() + FallbackDirectory.from_cache().values() + self._endpoints = [(directory.address, directory.dir_port) for directory in directories] if use_mirrors: try: @@ -412,8 +423,8 @@ class DescriptorDownloader(object): :raises: **Exception** if unable to determine the directory mirrors """ - authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values()) - new_endpoints = set([(auth.address, auth.dir_port) for auth in authorities]) + directories = get_authorities().values() + FallbackDirectory.from_cache().values() + new_endpoints = set([(directory.address, directory.dir_port) for directory in directories]) consensus = list(self.get_consensus(document_handler = stem.descriptor.DocumentHandler.DOCUMENT).run())[0] @@ -617,7 +628,39 @@ class DescriptorDownloader(object): ) -class DirectoryAuthority(object): +class Directory(object): + """ + Relay we can constact for directory information + + .. versionadded:: 1.5.0 + + :var str address: IP address of the authority, currently they're all IPv4 but + this may not always be the case + :var int or_port: port on which the relay services relay traffic + :var int dir_port: port on which directory information is available + :var str fingerprint: relay fingerprint + :var str nickname: nickname of the authority + """ + + def __init__(self, address, or_port, dir_port, fingerprint, nickname): + self.address = address + self.or_port = or_port + self.dir_port = dir_port + self.fingerprint = fingerprint + self.nickname = nickname + + def __eq__(self, other): + if not isinstance(other, Directory): + return False + + for attr in ('nickname', 'address', 'or_port', 'dir_port', 'fingerprint'): + if getattr(self, attr) != getattr(other, attr): + return False + + return True + + +class DirectoryAuthority(Directory): """ Tor directory authority, a special type of relay `hardcoded into tor <https://gitweb.torproject.org/tor.git/tree/src/or/config.c#n819>`_ @@ -648,23 +691,21 @@ class DirectoryAuthority(object): .. versionchanged:: 1.3.0 Added the is_bandwidth_authority attribute. - :var str nickname: nickname of the authority - :var str address: IP address of the authority, currently they're all IPv4 but - this may not always be the case - :var int or_port: port on which the relay services relay traffic - :var int dir_port: port on which directory information is available - :var str fingerprint: relay fingerprint :var str v3ident: identity key fingerprint used to sign votes and consensus + :var bool is_bandwidth_authority: **True** if this is a bandwidth authority, + **False** otherwise """ - def __init__(self, nickname = None, address = None, or_port = None, dir_port = None, is_bandwidth_authority = False, fingerprint = None, v3ident = None): - self.nickname = nickname - self.address = address - self.or_port = or_port - self.dir_port = dir_port - self.is_bandwidth_authority = is_bandwidth_authority - self.fingerprint = fingerprint + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): + super(DirectoryAuthority, self).__init__(address, or_port, dir_port, fingerprint, nickname) self.v3ident = v3ident + self.is_bandwidth_authority = is_bandwidth_authority + + def __eq__(self, other): + if isinstance(other, DirectoryAuthority) and super(DirectoryAuthority, self).__eq__(other): + return self.v3ident == other.v3ident and self.is_bandwidth_authority == other.is_bandwidth_authority + else: + return False DIRECTORY_AUTHORITIES = { @@ -767,7 +808,186 @@ def get_authorities(): The directory information hardcoded into Tor and occasionally changes, so the information this provides might not necessarily match your version of tor. - :returns: dict of str nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances + :returns: **dict** of **str** nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances """ return dict(DIRECTORY_AUTHORITIES) + + +class FallbackDirectory(Directory): + """ + Tor directories tor uses as alternates for the authorities. These relays are + `hardcoded in tor <https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc>`_. + + .. versionadded:: 1.5.0 + """ + + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None): + super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint, nickname) + + @staticmethod + def from_cache(): + """ + Provides fallback directory information cached with Stem. Unlike + :func:`~stem.descriptor.remote.FallbackDirectory.from_remote` this doesn't + have any system requirements, and is faster too. Only drawback is that + these fallback directories are only as up to date as the Stem release we're + using. + + :returns: **dict** of **str** fingerprints to their + :class:`~stem.descriptor.remote.FallbackDirectory` + """ + + conf = stem.util.conf.Config() + conf.load(CACHE_PATH) + + results = {} + + for nickname in set([key.split('.')[0] for key in conf.keys()]): + if nickname in ('tor_commit', 'stem_commit'): + continue + + attr = {} + + for attr_name in ('address', 'or_port', 'dir_port', 'fingerprint'): + key = '%s.%s' % (nickname, attr_name) + attr[attr_name] = conf.get(key) + + if not attr[attr_name]: + raise IOError("'%s' is missing from %s" % (key, CACHE_PATH)) + + if not connection.is_valid_ipv4_address(attr['address']): + raise IOError("'%s.address' was an invalid address (%s)" % (nickname, attr['address'])) + elif not connection.is_valid_port(attr['or_port']): + raise IOError("'%s.or_port' was an invalid port (%s)" % (nickname, attr['or_port'])) + elif not connection.is_valid_port(attr['dir_port']): + raise IOError("'%s.dir_port' was an invalid port (%s)" % (nickname, attr['dir_port'])) + elif not tor_tools.is_valid_fingerprint(attr['fingerprint']): + raise IOError("'%s.fingerprint' was an invalid fingerprint (%s)" % (nickname, attr['fingerprint'])) + + results[attr['fingerprint']] = FallbackDirectory( + address = attr['address'], + or_port = int(attr['or_port']), + dir_port = int(attr['dir_port']), + fingerprint = attr['fingerprint'], + nickname = nickname, + ) + + return results + + @staticmethod + def from_remote(timeout = 60): + """ + Reads and parses tor's latest fallback directories `from + gitweb.torproject.org + <https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc>`_. + Note that while convenient, this reliance on GitWeb means you should alway + call with a fallback, such as... + + :: + + try: + fallback_directories = stem.descriptor.remote.from_remote() + except IOError: + fallback_directories = stem.descriptor.remote.from_cache() + + :param int timeout: seconds to wait before timing out the request + + :returns: **dict** of **str** fingerprints to their + :class:`~stem.descriptor.remote.FallbackDirectory` + + :raises: **IOError** if unable to retrieve the fallback directories + """ + + try: + fallback_dir_page = urllib.urlopen(GITWEB_FALLBACK_DIR_URL, timeout = timeout).read() + except: + exc = sys.exc_info()[1] + raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc)) + + # Example of an entry... + # + # /* + # wagner + # Flags: Fast Guard Running Stable V2Dir Valid + # Fallback Weight: 43680 / 491920 (8.879%) + # Consensus Weight: 62600 / 546000 (11.465%) + # Rarely used email <trff914 AT gmail DOT com> + # */ + # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" + # " weight=43680", + + results, nickname, last_line = {}, None, None + + for line in fallback_dir_page.splitlines(): + if last_line == '/*': + nickname = line + elif line.startswith('"'): + addr_line_match = re.match('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) + + if addr_line_match: + address, dir_port, or_port, fingerprint = addr_line_match.groups() + + if not connection.is_valid_ipv4_address(address): + raise IOError('%s has an invalid address: %s' % (nickname, address)) + elif not connection.is_valid_port(or_port): + raise IOError('%s has an invalid or_port: %s' % (nickname, or_port)) + elif not connection.is_valid_port(dir_port): + raise IOError('%s has an invalid dir_port: %s' % (nickname, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise IOError('%s has an invalid fingerprint: %s' % (nickname, fingerprint)) + + results[fingerprint] = FallbackDirectory( + address = address, + or_port = int(or_port), + dir_port = int(dir_port), + fingerprint = fingerprint, + nickname = nickname, + ) + + last_line = line + + return results + + +def _fallback_directory_differences(previous_directories, new_directories): + """ + Provides a description of how fallback directories differ. + """ + + lines = [] + + added_fp = set(new_directories.keys()).difference(previous_directories.keys()) + removed_fp = set(previous_directories.keys()).difference(new_directories.keys()) + + for fp in added_fp: + directory = new_directories[fp] + + lines += [ + '* Added %s as a new fallback directory:' % directory.nickname, + ' address: %s' % directory.address, + ' or_port: %s' % directory.or_port, + ' dir_port: %s' % directory.dir_port, + ' fingerprint: %s' % directory.fingerprint, + '', + ] + + for fp in removed_fp: + lines.append('* Removed %s as a fallback directory' % previous_directories[fp].nickname) + + for fp in new_directories: + if fp in added_fp or fp in removed_fp: + continue # already discussed these + + previous_directory = previous_directories[fp] + new_directory = new_directories[fp] + + if previous_directory != new_directory: + for attr in ('nickname', 'address', 'or_port', 'dir_port', 'fingerprint'): + old_attr = getattr(previous_directory, attr) + new_attr = getattr(new_directory, attr) + + if old_attr != new_attr: + lines.append('* Changed the %s of %s from %s to %s' % (attr, fp, old_attr, new_attr)) + + return '\n'.join(lines) diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index fd37b57..9ce9b8f 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -218,3 +218,41 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertTrue(isinstance(single_query_results[0], stem.descriptor.networkstatus.KeyCertificate)) self.assertEqual(2, len(list(multiple_query))) + + @require_online + def test_that_cache_is_up_to_date(self): + """ + Check if the cached fallback directories bundled with Stem are up to date + or not. + """ + + cached_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() + latest_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() + + if cached_fallback_directories != latest_fallback_directories: + self.fail("Stem's cached fallback directories are out of date. Please run 'cache_fallback_directories.py'...\n\n%s" % stem.descriptor.remote._fallback_directory_differences(cached_fallback_directories, latest_fallback_directories)) + + @require_online + def test_that_fallback_directories_are_reachable(self): + """ + Fetch information from each fallback directory to confirm that it's + available. + """ + + unsuccessful = {} + downloader = stem.descriptor.remote.DescriptorDownloader() + moria1_v3ident = stem.descriptor.remote.get_authorities()['moria1'].v3ident + + for fallback_directory in stem.descriptor.remote.FallbackDirectory.from_cache().values(): + try: + downloader.get_key_certificates(authority_v3idents = moria1_v3ident, endpoints = [(fallback_directory.address, fallback_directory.dir_port)]).run() + except Exception as exc: + unsuccessful[fallback_directory] = exc + + if unsuccessful: + lines = ['We were unable to contact the following fallback directories...\n'] + + for fallback_directory, exc in unsuccessful.items(): + lines.append('* %s:%s (%s): %s' % (fallback_directory.address, fallback_directory.dir_port, fallback_directory.fingerprint, exc)) + + self.fail('\n'.join(lines)) diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 19907a7..f0e0dea 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -58,6 +58,32 @@ iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q -----END SIGNATURE----- """ +FALLBACK_DIR_CONTENT = b"""\ +/* Trial fallbacks for 0.2.8.1-alpha with ADDRESS_AND_PORT_STABLE_DAYS = 30 + * This works around an issue where relays post a descriptor without a DirPort + * when restarted. If these relays stay up, they will have been up for 120 days + * by the 0.2.8 stable release -- teor */ +/* +wagner +Flags: Fast Guard Running Stable V2Dir Valid +Fallback Weight: 43680 / 491920 (8.879%) +Consensus Weight: 62600 / 546000 (11.465%) +Rarely used email <trff914 AT gmail DOT com> +*/ +"5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" +" weight=43680", +/* +kitten2 +Flags: Fast Guard HSDir Running Stable V2Dir Valid +Fallback Weight: 43680 / 491920 (8.879%) +Consensus Weight: 59100 / 546000 (10.824%) +0xEFB74277ECE4E222 Aeris <aeris+tor AT imirhil DOT fr> - 1aerisnnLWPchhDSXpxWGYWwLiSFUVFnd +*/ +"62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E" +" ipv6=[2001:bc8:3f23:100::1]:9101" +" weight=43680", +""" + class TestDescriptorDownloader(unittest.TestCase): @patch(URL_OPEN) @@ -154,3 +180,33 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(1, len(list(query))) self.assertEqual(1, len(list(query))) self.assertEqual(1, len(list(query))) + + def test_fallback_directories_from_cache(self): + # quick sanity test that we can load cached content + fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() + self.assertTrue(len(fallback_directories) > 10) + self.assertEqual('wagner', fallback_directories['5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33'].nickname) + + @patch(URL_OPEN) + def test_fallback_directories_from_remote(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT) + fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() + + expected = { + '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33': stem.descriptor.remote.FallbackDirectory( + nickname = 'wagner', + address = '5.175.233.86', + or_port = 443, + dir_port = 80, + fingerprint = '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33', + ), + '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E': stem.descriptor.remote.FallbackDirectory( + nickname = 'kitten2', + address = '62.210.124.124', + or_port = 9101, + dir_port = 9130, + fingerprint = '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E', + ), + } + + self.assertEqual(expected, fallback_directories)

1 0

[doctor/master] Differentiate between 'no badexit flag' and 'not in consensus'
by atagar＠torproject.org 21 Feb '16

21 Feb '16

commit f1d40c6c736a72f9c607d5b170a0085b5f8053a0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 17:10:19 2016 -0800 Differentiate between 'no badexit flag' and 'not in consensus' Authorities usually flag bad actors by either BadExit or Rejection. As such when we're out of sync it might be because different authorities chose different methods. Updating our notice to better differentiate this... https://trac.torproject.org/projects/tor/ticket/17441#comment:3 --- consensus_health_checker.py | 21 +++++++++++++++++++-- data/consensus_health.cfg | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/consensus_health_checker.py b/consensus_health_checker.py index e8de5bf..68fad73 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -662,9 +662,26 @@ def bad_exits_in_sync(latest_consensus, consensuses, votes): for fingerprint in disagreed_bad_exits: with_flag = set([authority for authority, flagged in bad_exits.items() if fingerprint in flagged]) - without_flag = voting_authorities.difference(with_flag) + without_flag = [] + not_in_consensus = [] - issues.append(Issue(Runlevel.NOTICE, 'BADEXIT_OUT_OF_SYNC', fingerprint = fingerprint, with_flag = ', '.join(with_flag), without_flag = ', '.join(without_flag), to = bad_exits.keys())) + for authority in voting_authorities.difference(with_flag): + vote = votes[authority] + + if fingerprint in vote.routers: + without_flag.append(authority) + else: + not_in_consensus.append(authority) + + attr = ['with flag: %s' % ', '.join(with_flag)] + + if without_flag: + attr.append('without flag: %s' % ', '.join(without_flag)) + + if not_in_consensus: + attr.append('not in consensus: %s' % ', '.join(not_in_consensus)) + + issues.append(Issue(Runlevel.NOTICE, 'BADEXIT_OUT_OF_SYNC', fingerprint = fingerprint, counts = ', '.join(attr), to = bad_exits.keys())) return issues diff --git a/data/consensus_health.cfg b/data/consensus_health.cfg index 2c5652a..6d72cac 100644 --- a/data/consensus_health.cfg +++ b/data/consensus_health.cfg @@ -16,7 +16,7 @@ msg MISSING_AUTHORITIES => The following authorities are missing from the consen msg EXTRA_AUTHORITIES => The following authorities were not expected in the consensus: {authorities} msg FINGERPRINT_MISMATCH => {authority} had a different fingerprint than we expected (expected: {expected}, actual: {actual}) msg TOR_OUT_OF_DATE => The following authorities are an out of date version of tor: {authorities} -msg BADEXIT_OUT_OF_SYNC => Authorities disagree about the BadExit flag for {fingerprint} (with flag: {with_flag}, without flag: {without_flag}) +msg BADEXIT_OUT_OF_SYNC => Authorities disagree about the BadExit flag for {fingerprint} ({counts}) msg BANDWIDTH_AUTHORITIES_OUT_OF_SYNC => Bandwidth authorities have a substantially different number of measured entries: {authorities} msg AUTHORITY_UNAVAILABLE => Unable to retrieve the {fetch_type} from {authority} ({url}): {error}

1 0

[doctor/master] Re-enable check that BadExit flags are in sync
by atagar＠torproject.org 21 Feb '16

21 Feb '16

commit 8ada4e0305b16f6b854dae839db2adb7446b7d37 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 17:58:05 2016 -0800 Re-enable check that BadExit flags are in sync Roger says these are useful to him and that he's willing to invest effort to keep in sync so re-enabling the checks... https://trac.torproject.org/projects/tor/ticket/17441#comment:3 --- consensus_health_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus_health_checker.py b/consensus_health_checker.py index 68fad73..b0efb5c 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -323,7 +323,7 @@ def run_checks(consensuses, votes): #unmeasured_relays, has_authority_flag, is_recommended_versions, - #bad_exits_in_sync, + bad_exits_in_sync, bandwidth_authorities_in_sync, )

1 0

[torspec/master] Update prop264 per discussion
by nickm＠torproject.org 21 Feb '16

21 Feb '16

commit 4b772b32b46eca2cc021b9aab7bde4bdfb1d8498 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sat Feb 20 20:32:50 2016 -0500 Update prop264 per discussion --- proposals/264-subprotocol-versions.txt | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/proposals/264-subprotocol-versions.txt b/proposals/264-subprotocol-versions.txt index 20174b9..7d84228 100644 --- a/proposals/264-subprotocol-versions.txt +++ b/proposals/264-subprotocol-versions.txt @@ -125,18 +125,23 @@ Status: Open When a relay lacks a protocol listed in RequiredRelayProtocols, it must not attempt to join the network. + When a relay lacks a protocol listed in RecommendedRelayProtocols, it + should warn its operator that the relay is obsolete.. + When a client lacks a protocol listed in RecommendedClientProtocols, it should warn the user that the client is obsolete. When a client lacks a protocol listed in RequiredClientProtocols, it - must not use the corresponding feature. If the corresponding feature - is required to operate a Tor client, the client must must log an - error message and shut down without connecting to the network, even - if the cached consensus is out-of-date. This implements a "safe + must not connect to the network. This implements a "safe forward shutdown" mechanism for zombie clients. + If a client or relay has a cached consensus telling it that a given + protocol is required, and it does not implement that protocol, it + SHOULD NOT try to fetch a newer consensus. + The above features should be backported to 0.2.4 and later. + 5. Current protocols (See "6. Maintaining the protocol list" below for information about @@ -286,12 +291,12 @@ A.2. Initial required protocols For clients we will Recommend and Require: - DirCache=1 HSDir=1 Desc=1 Cons=ANYOF(9-20) Microdesc=ANYOF(9-20) + DirCache=1 HSDir=1 Desc=1 Cons=9-20 Microdesc=9-20 HSMid=1 Link=4 LinkAuth=1 Relay=2 For relays we will Require: - DirCache=1 HSDir=1 Desc=1 Cons=ANYOF(9-20) Microdesc=ANYOF(9-20) + DirCache=1 HSDir=1 Desc=1 Cons=9-20 Microdesc=9-20 HSMid=1 Link=3-4 LinkAuth=1 Relay=1-2 A.3. Example integration with other open proposals

1 0

[torspec/master] mark 265 accepted, targetting 029.
by nickm＠torproject.org 21 Feb '16

21 Feb '16

commit 1f07de13faaf997d4915e24837280f84a070c740 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sat Feb 20 20:17:23 2016 -0500 mark 265 accepted, targetting 029. --- proposals/000-index.txt | 4 ++-- proposals/265-load-balancing-with-overhead.txt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/proposals/000-index.txt b/proposals/000-index.txt index b5e3604..ae65d5a 100644 --- a/proposals/000-index.txt +++ b/proposals/000-index.txt @@ -185,7 +185,7 @@ Proposals by number: 262 Re-keying live circuits with new cryptographic material [OPEN] 263 Request to change key exchange protocol for handshake v1.2 [OPEN] 264 Putting version numbers on the Tor subprotocols [OPEN] -265 Load Balancing with Overhead Parameters [DRAFT] +265 Load Balancing with Overhead Parameters [ACCEPTED] 266 Removing current obsolete clients from the Tor network [DRAFT] @@ -212,7 +212,6 @@ Proposals by status: 255 Controller features to allow for load-balancing hidden services 259 New Guard Selection Behaviour 260 Rendezvous Single Onion Services - 265 Load Balancing with Overhead Parameters 266 Removing current obsolete clients from the Tor network NEEDS-REVISION: 190 Bridge Client Authorization Based on a Shared Secret @@ -253,6 +252,7 @@ Proposals by status: 220 Migrate server identity keys to Ed25519 [for 0.2.x.x] 244 Use RFC5705 Key Exporting in our AUTHENTICATE calls 258 Denial-of-service resistance for directory authorities + 265 Load Balancing with Overhead Parameters [for 0.2.9.x] META: 000 Index of Tor Proposals 001 The Tor Proposal Process diff --git a/proposals/265-load-balancing-with-overhead.txt b/proposals/265-load-balancing-with-overhead.txt index b1d5af6..4bf42a6 100644 --- a/proposals/265-load-balancing-with-overhead.txt +++ b/proposals/265-load-balancing-with-overhead.txt @@ -2,7 +2,8 @@ Filename: 265-load-balancing-with-overhead.txt Title: Load Balancing with Overhead Parameters Authors: Mike Perry Created: 01 January 2016 -Status: Draft +Status: Accepted +Target: 0.2.9.x 0. Motivation

1 0

[torspec/master] retitile 257 per discussion
by nickm＠torproject.org 21 Feb '16

21 Feb '16

commit 700c8a6f12cd7538844c21f629bdc74104a0d108 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sat Feb 20 20:16:02 2016 -0500 retitile 257 per discussion --- proposals/000-index.txt | 8 ++++---- proposals/257-hiding-authorities.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/proposals/000-index.txt b/proposals/000-index.txt index 20d6bb7..b5e3604 100644 --- a/proposals/000-index.txt +++ b/proposals/000-index.txt @@ -177,13 +177,13 @@ Proposals by number: 254 Padding Negotiation [DRAFT] 255 Controller features to allow for load-balancing hidden services [DRAFT] 256 Key revocation for relays and authorities [OPEN] -257 Refactoring authorities and taking parts offline [META] +257 Refactoring authorities and making them more isolated from the net [META] 258 Denial-of-service resistance for directory authorities [ACCEPTED] 259 New Guard Selection Behaviour [DRAFT] 260 Rendezvous Single Onion Services [DRAFT] 261 AEZ for relay cryptography [OPEN] 262 Re-keying live circuits with new cryptographic material [OPEN] -263 Request to change key exchange protocol for handshake v1.1 [OPEN] +263 Request to change key exchange protocol for handshake v1.2 [OPEN] 264 Putting version numbers on the Tor subprotocols [OPEN] 265 Load Balancing with Overhead Parameters [DRAFT] 266 Removing current obsolete clients from the Tor network [DRAFT] @@ -243,7 +243,7 @@ Proposals by status: 256 Key revocation for relays and authorities 261 AEZ for relay cryptography 262 Re-keying live circuits with new cryptographic material - 263 Request to change key exchange protocol for handshake v1.1 + 263 Request to change key exchange protocol for handshake v1.2 264 Putting version numbers on the Tor subprotocols ACCEPTED: 140 Provide diffs between consensuses @@ -258,7 +258,7 @@ Proposals by status: 001 The Tor Proposal Process 098 Proposals that should be written 099 Miscellaneous proposals - 257 Refactoring authorities and taking parts offline + 257 Refactoring authorities and making them more isolated from the net FINISHED: 121 Hidden Service Authentication [in 0.2.1.x] 151 Improving Tor Path Selection diff --git a/proposals/257-hiding-authorities.txt b/proposals/257-hiding-authorities.txt index cab85ba..9c32f90 100644 --- a/proposals/257-hiding-authorities.txt +++ b/proposals/257-hiding-authorities.txt @@ -1,5 +1,5 @@ Filename: 257-hiding-authorities.txt -Title: Refactoring authorities and taking parts offline +Title: Refactoring authorities and making them more isolated from the net Authors: Nick Mathewson, Andrea Shepard Created: 2015-10-27 Status: Meta

1 0

[doctor/master] Move TEST_RUN to util
by atagar＠torproject.org 20 Feb '16

20 Feb '16

commit 63eb5d36b8ee5f4b23d879482e1214e60c58e65a Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Feb 18 08:08:19 2016 -0800 Move TEST_RUN to util To test locally without spamming lists we have a simple TEST_RUN flag that prints results rather than emailing. Moving this into the util so all our scripts can take advantage of this. --- consensus_health_checker.py | 29 +++++++++-------------------- descriptor_checker.py | 2 +- fingerprint_change_checker.py | 2 +- sybil_checker.py | 2 +- util.py | 9 +++++++++ 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/consensus_health_checker.py b/consensus_health_checker.py index 1962d10..e8de5bf 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2013, Damian Johnson and The Tor Project +# Copyright 2013-2016, Damian Johnson and The Tor Project # See LICENSE for licensing information """ @@ -23,10 +23,6 @@ import stem.util.enum from stem import Flag from stem.util.lru_cache import lru_cache -# Set this flag to print results rather than email them. - -TEST_RUN = False - Runlevel = stem.util.enum.UppercaseEnum('NOTICE', 'WARNING', 'ERROR') DIRECTORY_AUTHORITIES = stem.descriptor.remote.get_authorities() @@ -276,19 +272,16 @@ def main(): log.debug('Sending notification for issues (%s)' % ', '.join(destination_labels)) - if TEST_RUN: - print('\n'.join(map(str, issues))) - else: - body = '\n'.join(map(str, issues)) - cc = [d.address for d in destinations.values() if d and not d.bcc] - bcc = [d.address for d in destinations.values() if d and d.bcc] + body = '\n'.join(map(str, issues)) + cc = [d.address for d in destinations.values() if d and not d.bcc] + bcc = [d.address for d in destinations.values() if d and d.bcc] - util.send(EMAIL_SUBJECT, body = body, cc = cc, bcc = bcc) + util.send(EMAIL_SUBJECT, body = body, cc = cc, bcc = bcc) - # notification for #tor-bots + # notification for #tor-bots - body = '\n'.join(['[consensus-health] %s' % issue for issue in issues]) - util.send('Announce or', body = body, to = ['tor-misc(a)commit.noreply.org']) + body = '\n'.join(['[consensus-health] %s' % issue for issue in issues]) + util.send('Announce or', body = body, to = ['tor-misc(a)commit.noreply.org']) else: if issues: log.info("All %i issues were suppressed. Not sending a notification." % len(issues)) @@ -795,8 +788,4 @@ if __name__ == '__main__': except: msg = "consensus_health_checker.py failed with:\n\n%s" % traceback.format_exc() log.error(msg) - - if TEST_RUN: - print("Error: %s" % msg) - else: - util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS]) + util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS]) diff --git a/descriptor_checker.py b/descriptor_checker.py index 05a6513..953def1 100755 --- a/descriptor_checker.py +++ b/descriptor_checker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2013, Damian Johnson and The Tor Project +# Copyright 2013-2016, Damian Johnson and The Tor Project # See LICENSE for licensing information """ diff --git a/fingerprint_change_checker.py b/fingerprint_change_checker.py index c6a4b98..493f51c 100755 --- a/fingerprint_change_checker.py +++ b/fingerprint_change_checker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2015, Damian Johnson and The Tor Project +# Copyright 2015-2016, Damian Johnson and The Tor Project # See LICENSE for licensing information """ diff --git a/sybil_checker.py b/sybil_checker.py index 52fba0c..daaf0bd 100755 --- a/sybil_checker.py +++ b/sybil_checker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2013, Damian Johnson and The Tor Project +# Copyright 2013-2016, Damian Johnson and The Tor Project # See LICENSE for licensing information """ diff --git a/util.py b/util.py index 9440d4c..483181e 100644 --- a/util.py +++ b/util.py @@ -15,6 +15,8 @@ FROM_ADDRESS = 'atagar(a)torproject.org' TO_ADDRESSES = ['tor-consensus-health(a)lists.torproject.org'] ERROR_ADDRESS = 'atagar(a)torproject.org' +TEST_RUN = False # print script results rather than emailing + def get_path(*comp): """ @@ -88,6 +90,13 @@ def send(subject, body, to = TO_ADDRESSES, cc = None, bcc = None): :raises: **Exception** if the email fails to be sent """ + if TEST_RUN: + print('Email to: %s' % to) + print('Subject: %s' % subject) + print('-' * 60) + print(body) + return + msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = FROM_ADDRESS

1 0

[doctor/master] Support address ranges for relays we track
by atagar＠torproject.org 20 Feb '16

20 Feb '16

commit 44b3a702b34604d67cfaa0da50e5ae243e5c41c2 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 12:42:54 2016 -0800 Support address ranges for relays we track Ewww, but Stem's ExitPolicy already supports exactly what we want in a delightfully efficient way so taking advantage of that. --- track_relays.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/track_relays.py b/track_relays.py index a65356d..bc46216 100755 --- a/track_relays.py +++ b/track_relays.py @@ -10,6 +10,7 @@ import datetime import traceback import stem.descriptor.remote +import stem.exit_policy import stem.util.conf import util @@ -111,7 +112,10 @@ def main(): for relay in get_tracked_relays(): if relay.address: if '/' in relay.address: - tracked_address_ranges[relay.address] = relay + # It's a total hack, but taking advantage of exit policies where we + # already support address ranges. + + tracked_address_ranges[stem.exit_policy.ExitPolicyRule('accept %s:*' % relay.address)] = relay else: tracked_addresses[relay.address] = relay @@ -127,7 +131,9 @@ def main(): elif desc.fingerprint in tracked_fingerprints: found_relays[tracked_fingerprints[desc.fingerprint]] = desc else: - pass # TODO: implement for tracked_address_ranges + for addr_entry, relay in tracked_address_ranges.items(): + if addr_entry.is_match(desc.address): + found_relays[relay] = desc if found_relays: log.debug("Sending a notification for %i relay entries..." % len(found_relays))

1 0

[doctor/master] Track when malicious relays return to the network
by atagar＠torproject.org 20 Feb '16

20 Feb '16

commit efa0ed43194bbc833ad91233586c40198070a94f Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 11:36:42 2016 -0800 Track when malicious relays return to the network When BadExit relays are removed we want to continue to monitor for their return for a time after. This is a script requested by David on... https://trac.torproject.org/projects/tor/ticket/18246 Basic script's done but as the TODO comments show there's still some missing bits. --- data/tracked_relays.cfg | 22 +++++++ fingerprint_change_checker.py | 5 +- track_relays.py | 136 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 4 deletions(-) diff --git a/data/tracked_relays.cfg b/data/tracked_relays.cfg new file mode 100644 index 0000000..e25959e --- /dev/null +++ b/data/tracked_relays.cfg @@ -0,0 +1,22 @@ +# Relays we want notifications for when they reappear. This can be done by +# address (single addresses or ranges) or fingerprints. +# +# Anything in this file is public. All entries *MUST* have a description and an +# expiration date for when it can be removed. The description is particularly +# important so we can puzzle out why we were monitoring for this if it +# reappears. +# +# Few examples... +# +# PrivacyPT.description running sslstrip on 2011-01-05 +# PrivacyPT.expires 2016-01-15 +# PrivacyPT.address 84.90.72.186 +# +# trotsky.description sybil consisting of 383 exit relays on 2010-10-02 +# trotsky.expires 2016-02-01 +# trotsky.address 185.19.80.0/26 +# +# Unnamed001.description harvesting hidden services on 2012-08-12 +# Unnamed001.expires 2016-03-01 +# Unnamed001.fingerprint 05AF83344B3787D0DCCD47DC4A6A4668142A5F8C + diff --git a/fingerprint_change_checker.py b/fingerprint_change_checker.py index 493f51c..e13f3d0 100755 --- a/fingerprint_change_checker.py +++ b/fingerprint_change_checker.py @@ -89,10 +89,7 @@ def main(): body += "\n" - try: - util.send(EMAIL_SUBJECT, body = body, to = ['bad-relays(a)lists.torproject.org', 'atagar(a)torproject.org']) - except Exception as exc: - log.warn("Unable to send email: %s" % exc) + util.send(EMAIL_SUBJECT, body = body, to = ['bad-relays(a)lists.torproject.org', 'atagar(a)torproject.org']) # register that we've notified for these diff --git a/track_relays.py b/track_relays.py new file mode 100755 index 0000000..e770006 --- /dev/null +++ b/track_relays.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# Copyright 2016, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Notifies if specific relays reappear in the network. +""" + +import datetime +import traceback + +import stem.descriptor.remote +import stem.util.conf + +import util + +log = util.get_logger('track_relays') + +EMAIL_SUBJECT = 'Relays Returned' + +EMAIL_BODY = """\ +The following previously BadExit relays have returned to the network... + +""" + + +class TrackedRelay(object): + """ + Represents a relay we're keeping an eye on. + + :var str identifier: brief identifier given to the entry + :var str description: description of why we're tracking it + :var datetime expires: when this entry expires + :var str address: address of the relay we're tracking + :var str fingerprint: fingerprint of the relay we're tracking + """ + + def __init__(self, identifier, config): + self.identifier = identifier + self.description = config.get('%s.description' % identifier, '') + + expires_str = config.get('%s.expires' % identifier, '') + + if not expires_str: + raise ValueError("Our config file is missing a '%s.expires' entry" % identifier) + + try: + self.expires = datetime.datetime.strptime(expires_str, '%Y-%m-%d') + except ValueError: + raise ValueError("'%s.expires' is malformed. We expect it to be in the form 'Year-Month-Day'" % identifier) + + self.address = config.get('%s.address' % identifier, None) + self.fingerprint = config.get('%s.fingerprint' % identifier, None) + + if not self.address and not self.fingerprint: + raise ValueError("We need either a '%s.address' or '%s.fingerprint' to track" % (identifier, identifier)) + + def __str__(self): + attr = [] + + if self.address: + attr.append('address: %s' % self.address) + + if self.fingerprint: + attr.append('fingerprint: %s' % self.fingerprint) + + return '%s (%s)' % (self.identifier, ', '.join(attr)) + + +def get_tracked_relays(): + """ + Provides the relays we're tracking. + + :returns: **list** of **TrackedRelay** we're tracking + + :raises: **ValueError** if our config file is malformed + """ + + config = stem.util.conf.get_config('tracked_relays') + config.load(util.get_path('data', 'tracked_relays.cfg')) + + # TODO: check for expired entries + + identifiers = set([key.split('.')[0] for key in config.keys()]) + return [TrackedRelay(identifier, config) for identifier in identifiers] + + +def main(): + # Map addresses and fingerprints to relays for constant time lookups. Address + # ranges are handled separately cuz... well, they're a pita. + + tracked_addresses = {} + tracked_address_ranges = {} + tracked_fingerprints = {} + + for relay in get_tracked_relays(): + if relay.address: + if '/' in relay.address: + tracked_address_ranges[relay.address] = relay + else: + tracked_addresses[relay.address] = relay + + if relay.fingerprint: + tracked_fingerprints[relay.fingerprint] = relay + + downloader = stem.descriptor.remote.DescriptorDownloader() + found_relays = {} # mapping of TrackedRelay => RouterStatusEntry + + for desc in downloader.get_consensus(): + if desc.address in tracked_addresses: + found_relays[tracked_addresses[desc.address]] = desc + elif desc.fingerprint in tracked_fingerprints: + found_relays[tracked_fingerprints[desc.fingerprint]] = desc + else: + pass # TODO: implement for tracked_address_ranges + + if found_relays: + log.debug("Sending a notification for %i relay entries..." % len(found_relays)) + body = EMAIL_BODY + + for tracked_relay, desc in found_relays.items(): + log.debug('* %s' % tracked_relay) + body += '* %s (%s)\n' % (tracked_relay.identifier, tracked_relay.description) + body += ' address: %s\n' % desc.address + body += ' fingerprint: %s\n\n' % desc.fingerprint + + util.send(EMAIL_SUBJECT, body = body, to = ['bad-relays(a)lists.torproject.org', 'atagar(a)torproject.org']) + + +if __name__ == '__main__': + try: + main() + except: + msg = "track_relays.py failed with:\n\n%s" % traceback.format_exc() + log.error(msg) + util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS])

1 0