February 2016 - tor-commits - lists.torproject.org

[stem/master] Fallback directory information
by atagar＠torproject.org 22 Feb '16

22 Feb '16

commit 332ef6e6fe89d9f0d48e9b95acff7de9f3ec647f Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 21 18:04:40 2016 -0800 Fallback directory information Recently tor added fallback directory authorities to help clients bootstrap when the dirauths are unavailable... https://trac.torproject.org/projects/tor/ticket/17158 To help support this effort teor asked for a DocTor check to notify us when the directories become unavailable... … [View More] https://trac.torproject.org/projects/tor/ticket/18177 Great idea but I'm expanding this. Stem now has a FallbackDirectory class with two methods for getting this information... * FallbackDirectory.from_remote() - Reads the latest fallback_dirs.inc from gitweb, providing the latest fallback directories in tor's master branch. * FallbackDirectory.from_cache() - Provides the latest fallback directories Stem has cached. This is only as up-to-date as the Stem release you're using but is quicker and avoids relying on gitweb. Advantages are... * Stem's descriptor.remote module now puts less load on the directory authorities since it uses fallback directories as well. * Running Stem's integ tests with the ONLINE target includes a test that exercises all the fallback directories, notifying us if any are down. --- cache_manual.py | 4 +- docs/change_log.rst | 1 + run_tests.py | 1 + setup.py | 2 +- stem/descriptor/remote.py | 266 ++++++++++++++++++++++++++++++++++++---- test/integ/descriptor/remote.py | 38 ++++++ test/unit/descriptor/remote.py | 56 +++++++++ 7 files changed, 341 insertions(+), 27 deletions(-) diff --git a/cache_manual.py b/cache_manual.py index aa55dc5..19e73f0 100755 --- a/cache_manual.py +++ b/cache_manual.py @@ -6,7 +6,6 @@ Caches tor's latest manual content. Run this to pick new man page changes. """ -import os import re import sys @@ -19,7 +18,6 @@ try: except ImportError: import urllib2 as urllib -CACHE_PATH = os.path.join(os.path.dirname(__file__), 'stem', 'cached_tor_manual.cfg') GITWEB_MAN_LOG = 'https://gitweb.torproject.org/tor.git/log/doc/tor.1.txt' MAN_LOG_LINK = "href='/tor.git/commit/doc/tor.1.txt\?id=([^']*)'" @@ -53,4 +51,4 @@ if __name__ == '__main__': latest_manual.man_commit = man_commit latest_manual.stem_commit = stem_commit - latest_manual.save(CACHE_PATH) + latest_manual.save(stem.manual.CACHE_PATH) diff --git a/docs/change_log.rst b/docs/change_log.rst index 81bf5a2..c8dc434 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -58,6 +58,7 @@ The following are only available within Stem's `git repository * **Descriptors** + * Fallback directory information, lessing the load of `stem.descriptor.remote <api/descriptor/remote.html>`_ on the directory authorities * Support for ed25519 descriptor fields (:spec:`5a79d67`) * Server descriptor validation fails with 'extra-info-digest line had an invalid value' from additions in proposal 228 (:trac:`16227`) * :class:`~stem.descriptor.server_descriptor.BridgeDescriptor` now has 'ntor_onion_key' like its unsanitized counterparts diff --git a/run_tests.py b/run_tests.py index 3d0f845..2fb93bf 100755 --- a/run_tests.py +++ b/run_tests.py @@ -42,6 +42,7 @@ SRC_PATHS = [os.path.join(STEM_BASE, path) for path in ( 'test', 'run_tests.py', 'cache_manual.py', + 'cache_fallback_directories.py', 'setup.py', 'tor-prompt', os.path.join('docs', 'republish.py'), diff --git a/setup.py b/setup.py index 2a78b9b..2e492d4 100644 --- a/setup.py +++ b/setup.py @@ -17,5 +17,5 @@ distutils.core.setup( packages = ['stem', 'stem.descriptor', 'stem.interpreter', 'stem.response', 'stem.util'], keywords = 'tor onion controller', scripts = ['tor-prompt'], - package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']}, + package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.descriptor': ['fallback_directories.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']}, ) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index d199fb2..dc40e41 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -44,7 +44,11 @@ itself... get_authorities - Provides tor directory information. - DirectoryAuthority - Information about a tor directory authority. + Directory - Relay we can retrieve directory information from + |- DirectoryAuthority - Information about a tor directory authority + +- FallbackDirectory - Directory mirror tor uses when authories are unavailable + |- from_cache - Provides fallback directories cached with Stem. + +- from_remote - Retrieves fallback directories remotely from tor's latest commit. Query - Asynchronous request to download tor descriptors |- start - issues the query if it isn't already running @@ -73,7 +77,9 @@ itself... """ import io +import os import random +import re import sys import threading import time @@ -88,7 +94,7 @@ except ImportError: import stem.descriptor from stem import Flag -from stem.util import log +from stem.util import connection, log, tor_tools # Tor has a limited number of descriptors we can fetch explicitly by their # fingerprint or hashes due to a limit on the url length by squid proxies. @@ -96,6 +102,9 @@ from stem.util import log MAX_FINGERPRINTS = 96 MAX_MICRODESCRIPTOR_HASHES = 92 +GITWEB_FALLBACK_DIR_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' +CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg') + def _guess_descriptor_type(resource): # Attempts to determine the descriptor type based on the resource url. This @@ -340,8 +349,10 @@ class Query(object): """ if use_authority or not self.endpoints: - authority = random.choice(list(filter(lambda auth: auth.v3ident is not None, get_authorities().values()))) - address, dirport = authority.address, authority.dir_port + directories = get_authorities().values() + FallbackDirectory.from_cache().values() + + picked = random.choice(directories) + address, dirport = picked.address, picked.dir_port else: address, dirport = random.choice(self.endpoints) @@ -390,8 +401,8 @@ class DescriptorDownloader(object): def __init__(self, use_mirrors = False, **default_args): self._default_args = default_args - authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values()) - self._endpoints = [(auth.address, auth.dir_port) for auth in authorities] + directories = get_authorities().values() + FallbackDirectory.from_cache().values() + self._endpoints = [(directory.address, directory.dir_port) for directory in directories] if use_mirrors: try: @@ -412,8 +423,8 @@ class DescriptorDownloader(object): :raises: **Exception** if unable to determine the directory mirrors """ - authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values()) - new_endpoints = set([(auth.address, auth.dir_port) for auth in authorities]) + directories = get_authorities().values() + FallbackDirectory.from_cache().values() + new_endpoints = set([(directory.address, directory.dir_port) for directory in directories]) consensus = list(self.get_consensus(document_handler = stem.descriptor.DocumentHandler.DOCUMENT).run())[0] @@ -617,7 +628,39 @@ class DescriptorDownloader(object): ) -class DirectoryAuthority(object): +class Directory(object): + """ + Relay we can constact for directory information + + .. versionadded:: 1.5.0 + + :var str address: IP address of the authority, currently they're all IPv4 but + this may not always be the case + :var int or_port: port on which the relay services relay traffic + :var int dir_port: port on which directory information is available + :var str fingerprint: relay fingerprint + :var str nickname: nickname of the authority + """ + + def __init__(self, address, or_port, dir_port, fingerprint, nickname): + self.address = address + self.or_port = or_port + self.dir_port = dir_port + self.fingerprint = fingerprint + self.nickname = nickname + + def __eq__(self, other): + if not isinstance(other, Directory): + return False + + for attr in ('nickname', 'address', 'or_port', 'dir_port', 'fingerprint'): + if getattr(self, attr) != getattr(other, attr): + return False + + return True + + +class DirectoryAuthority(Directory): """ Tor directory authority, a special type of relay `hardcoded into tor <https://gitweb.torproject.org/tor.git/tree/src/or/config.c#n819>`_ @@ -648,23 +691,21 @@ class DirectoryAuthority(object): .. versionchanged:: 1.3.0 Added the is_bandwidth_authority attribute. - :var str nickname: nickname of the authority - :var str address: IP address of the authority, currently they're all IPv4 but - this may not always be the case - :var int or_port: port on which the relay services relay traffic - :var int dir_port: port on which directory information is available - :var str fingerprint: relay fingerprint :var str v3ident: identity key fingerprint used to sign votes and consensus + :var bool is_bandwidth_authority: **True** if this is a bandwidth authority, + **False** otherwise """ - def __init__(self, nickname = None, address = None, or_port = None, dir_port = None, is_bandwidth_authority = False, fingerprint = None, v3ident = None): - self.nickname = nickname - self.address = address - self.or_port = or_port - self.dir_port = dir_port - self.is_bandwidth_authority = is_bandwidth_authority - self.fingerprint = fingerprint + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): + super(DirectoryAuthority, self).__init__(address, or_port, dir_port, fingerprint, nickname) self.v3ident = v3ident + self.is_bandwidth_authority = is_bandwidth_authority + + def __eq__(self, other): + if isinstance(other, DirectoryAuthority) and super(DirectoryAuthority, self).__eq__(other): + return self.v3ident == other.v3ident and self.is_bandwidth_authority == other.is_bandwidth_authority + else: + return False DIRECTORY_AUTHORITIES = { @@ -767,7 +808,186 @@ def get_authorities(): The directory information hardcoded into Tor and occasionally changes, so the information this provides might not necessarily match your version of tor. - :returns: dict of str nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances + :returns: **dict** of **str** nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances """ return dict(DIRECTORY_AUTHORITIES) + + +class FallbackDirectory(Directory): + """ + Tor directories tor uses as alternates for the authorities. These relays are + `hardcoded in tor <https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc>`_. + + .. versionadded:: 1.5.0 + """ + + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None): + super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint, nickname) + + @staticmethod + def from_cache(): + """ + Provides fallback directory information cached with Stem. Unlike + :func:`~stem.descriptor.remote.FallbackDirectory.from_remote` this doesn't + have any system requirements, and is faster too. Only drawback is that + these fallback directories are only as up to date as the Stem release we're + using. + + :returns: **dict** of **str** fingerprints to their + :class:`~stem.descriptor.remote.FallbackDirectory` + """ + + conf = stem.util.conf.Config() + conf.load(CACHE_PATH) + + results = {} + + for nickname in set([key.split('.')[0] for key in conf.keys()]): + if nickname in ('tor_commit', 'stem_commit'): + continue + + attr = {} + + for attr_name in ('address', 'or_port', 'dir_port', 'fingerprint'): + key = '%s.%s' % (nickname, attr_name) + attr[attr_name] = conf.get(key) + + if not attr[attr_name]: + raise IOError("'%s' is missing from %s" % (key, CACHE_PATH)) + + if not connection.is_valid_ipv4_address(attr['address']): + raise IOError("'%s.address' was an invalid address (%s)" % (nickname, attr['address'])) + elif not connection.is_valid_port(attr['or_port']): + raise IOError("'%s.or_port' was an invalid port (%s)" % (nickname, attr['or_port'])) + elif not connection.is_valid_port(attr['dir_port']): + raise IOError("'%s.dir_port' was an invalid port (%s)" % (nickname, attr['dir_port'])) + elif not tor_tools.is_valid_fingerprint(attr['fingerprint']): + raise IOError("'%s.fingerprint' was an invalid fingerprint (%s)" % (nickname, attr['fingerprint'])) + + results[attr['fingerprint']] = FallbackDirectory( + address = attr['address'], + or_port = int(attr['or_port']), + dir_port = int(attr['dir_port']), + fingerprint = attr['fingerprint'], + nickname = nickname, + ) + + return results + + @staticmethod + def from_remote(timeout = 60): + """ + Reads and parses tor's latest fallback directories `from + gitweb.torproject.org + <https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc>`_. + Note that while convenient, this reliance on GitWeb means you should alway + call with a fallback, such as... + + :: + + try: + fallback_directories = stem.descriptor.remote.from_remote() + except IOError: + fallback_directories = stem.descriptor.remote.from_cache() + + :param int timeout: seconds to wait before timing out the request + + :returns: **dict** of **str** fingerprints to their + :class:`~stem.descriptor.remote.FallbackDirectory` + + :raises: **IOError** if unable to retrieve the fallback directories + """ + + try: + fallback_dir_page = urllib.urlopen(GITWEB_FALLBACK_DIR_URL, timeout = timeout).read() + except: + exc = sys.exc_info()[1] + raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc)) + + # Example of an entry... + # + # /* + # wagner + # Flags: Fast Guard Running Stable V2Dir Valid + # Fallback Weight: 43680 / 491920 (8.879%) + # Consensus Weight: 62600 / 546000 (11.465%) + # Rarely used email <trff914 AT gmail DOT com> + # */ + # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" + # " weight=43680", + + results, nickname, last_line = {}, None, None + + for line in fallback_dir_page.splitlines(): + if last_line == '/*': + nickname = line + elif line.startswith('"'): + addr_line_match = re.match('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) + + if addr_line_match: + address, dir_port, or_port, fingerprint = addr_line_match.groups() + + if not connection.is_valid_ipv4_address(address): + raise IOError('%s has an invalid address: %s' % (nickname, address)) + elif not connection.is_valid_port(or_port): + raise IOError('%s has an invalid or_port: %s' % (nickname, or_port)) + elif not connection.is_valid_port(dir_port): + raise IOError('%s has an invalid dir_port: %s' % (nickname, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise IOError('%s has an invalid fingerprint: %s' % (nickname, fingerprint)) + + results[fingerprint] = FallbackDirectory( + address = address, + or_port = int(or_port), + dir_port = int(dir_port), + fingerprint = fingerprint, + nickname = nickname, + ) + + last_line = line + + return results + + +def _fallback_directory_differences(previous_directories, new_directories): + """ + Provides a description of how fallback directories differ. + """ + + lines = [] + + added_fp = set(new_directories.keys()).difference(previous_directories.keys()) + removed_fp = set(previous_directories.keys()).difference(new_directories.keys()) + + for fp in added_fp: + directory = new_directories[fp] + + lines += [ + '* Added %s as a new fallback directory:' % directory.nickname, + ' address: %s' % directory.address, + ' or_port: %s' % directory.or_port, + ' dir_port: %s' % directory.dir_port, + ' fingerprint: %s' % directory.fingerprint, + '', + ] + + for fp in removed_fp: + lines.append('* Removed %s as a fallback directory' % previous_directories[fp].nickname) + + for fp in new_directories: + if fp in added_fp or fp in removed_fp: + continue # already discussed these + + previous_directory = previous_directories[fp] + new_directory = new_directories[fp] + + if previous_directory != new_directory: + for attr in ('nickname', 'address', 'or_port', 'dir_port', 'fingerprint'): + old_attr = getattr(previous_directory, attr) + new_attr = getattr(new_directory, attr) + + if old_attr != new_attr: + lines.append('* Changed the %s of %s from %s to %s' % (attr, fp, old_attr, new_attr)) + + return '\n'.join(lines) diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index fd37b57..9ce9b8f 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -218,3 +218,41 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertTrue(isinstance(single_query_results[0], stem.descriptor.networkstatus.KeyCertificate)) self.assertEqual(2, len(list(multiple_query))) + + @require_online + def test_that_cache_is_up_to_date(self): + """ + Check if the cached fallback directories bundled with Stem are up to date + or not. + """ + + cached_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() + latest_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() + + if cached_fallback_directories != latest_fallback_directories: + self.fail("Stem's cached fallback directories are out of date. Please run 'cache_fallback_directories.py'...\n\n%s" % stem.descriptor.remote._fallback_directory_differences(cached_fallback_directories, latest_fallback_directories)) + + @require_online + def test_that_fallback_directories_are_reachable(self): + """ + Fetch information from each fallback directory to confirm that it's + available. + """ + + unsuccessful = {} + downloader = stem.descriptor.remote.DescriptorDownloader() + moria1_v3ident = stem.descriptor.remote.get_authorities()['moria1'].v3ident + + for fallback_directory in stem.descriptor.remote.FallbackDirectory.from_cache().values(): + try: + downloader.get_key_certificates(authority_v3idents = moria1_v3ident, endpoints = [(fallback_directory.address, fallback_directory.dir_port)]).run() + except Exception as exc: + unsuccessful[fallback_directory] = exc + + if unsuccessful: + lines = ['We were unable to contact the following fallback directories...\n'] + + for fallback_directory, exc in unsuccessful.items(): + lines.append('* %s:%s (%s): %s' % (fallback_directory.address, fallback_directory.dir_port, fallback_directory.fingerprint, exc)) + + self.fail('\n'.join(lines)) diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 19907a7..f0e0dea 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -58,6 +58,32 @@ iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q -----END SIGNATURE----- """ +FALLBACK_DIR_CONTENT = b"""\ +/* Trial fallbacks for 0.2.8.1-alpha with ADDRESS_AND_PORT_STABLE_DAYS = 30 + * This works around an issue where relays post a descriptor without a DirPort + * when restarted. If these relays stay up, they will have been up for 120 days + * by the 0.2.8 stable release -- teor */ +/* +wagner +Flags: Fast Guard Running Stable V2Dir Valid +Fallback Weight: 43680 / 491920 (8.879%) +Consensus Weight: 62600 / 546000 (11.465%) +Rarely used email <trff914 AT gmail DOT com> +*/ +"5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" +" weight=43680", +/* +kitten2 +Flags: Fast Guard HSDir Running Stable V2Dir Valid +Fallback Weight: 43680 / 491920 (8.879%) +Consensus Weight: 59100 / 546000 (10.824%) +0xEFB74277ECE4E222 Aeris <aeris+tor AT imirhil DOT fr> - 1aerisnnLWPchhDSXpxWGYWwLiSFUVFnd +*/ +"62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E" +" ipv6=[2001:bc8:3f23:100::1]:9101" +" weight=43680", +""" + class TestDescriptorDownloader(unittest.TestCase): @patch(URL_OPEN) @@ -154,3 +180,33 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(1, len(list(query))) self.assertEqual(1, len(list(query))) self.assertEqual(1, len(list(query))) + + def test_fallback_directories_from_cache(self): + # quick sanity test that we can load cached content + fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() + self.assertTrue(len(fallback_directories) > 10) + self.assertEqual('wagner', fallback_directories['5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33'].nickname) + + @patch(URL_OPEN) + def test_fallback_directories_from_remote(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT) + fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() + + expected = { + '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33': stem.descriptor.remote.FallbackDirectory( + nickname = 'wagner', + address = '5.175.233.86', + or_port = 443, + dir_port = 80, + fingerprint = '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33', + ), + '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E': stem.descriptor.remote.FallbackDirectory( + nickname = 'kitten2', + address = '62.210.124.124', + or_port = 9101, + dir_port = 9130, + fingerprint = '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E', + ), + } + + self.assertEqual(expected, fallback_directories) [View Less]

1 0

[doctor/master] Differentiate between 'no badexit flag' and 'not in consensus'
by atagar＠torproject.org 21 Feb '16

21 Feb '16

commit f1d40c6c736a72f9c607d5b170a0085b5f8053a0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 17:10:19 2016 -0800 Differentiate between 'no badexit flag' and 'not in consensus' Authorities usually flag bad actors by either BadExit or Rejection. As such when we're out of sync it might be because different authorities chose different methods. Updating our notice to better differentiate this... https://trac.torproject.org/projects/tor/… [View More]ticket/17441#comment:3 --- consensus_health_checker.py | 21 +++++++++++++++++++-- data/consensus_health.cfg | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/consensus_health_checker.py b/consensus_health_checker.py index e8de5bf..68fad73 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -662,9 +662,26 @@ def bad_exits_in_sync(latest_consensus, consensuses, votes): for fingerprint in disagreed_bad_exits: with_flag = set([authority for authority, flagged in bad_exits.items() if fingerprint in flagged]) - without_flag = voting_authorities.difference(with_flag) + without_flag = [] + not_in_consensus = [] - issues.append(Issue(Runlevel.NOTICE, 'BADEXIT_OUT_OF_SYNC', fingerprint = fingerprint, with_flag = ', '.join(with_flag), without_flag = ', '.join(without_flag), to = bad_exits.keys())) + for authority in voting_authorities.difference(with_flag): + vote = votes[authority] + + if fingerprint in vote.routers: + without_flag.append(authority) + else: + not_in_consensus.append(authority) + + attr = ['with flag: %s' % ', '.join(with_flag)] + + if without_flag: + attr.append('without flag: %s' % ', '.join(without_flag)) + + if not_in_consensus: + attr.append('not in consensus: %s' % ', '.join(not_in_consensus)) + + issues.append(Issue(Runlevel.NOTICE, 'BADEXIT_OUT_OF_SYNC', fingerprint = fingerprint, counts = ', '.join(attr), to = bad_exits.keys())) return issues diff --git a/data/consensus_health.cfg b/data/consensus_health.cfg index 2c5652a..6d72cac 100644 --- a/data/consensus_health.cfg +++ b/data/consensus_health.cfg @@ -16,7 +16,7 @@ msg MISSING_AUTHORITIES => The following authorities are missing from the consen msg EXTRA_AUTHORITIES => The following authorities were not expected in the consensus: {authorities} msg FINGERPRINT_MISMATCH => {authority} had a different fingerprint than we expected (expected: {expected}, actual: {actual}) msg TOR_OUT_OF_DATE => The following authorities are an out of date version of tor: {authorities} -msg BADEXIT_OUT_OF_SYNC => Authorities disagree about the BadExit flag for {fingerprint} (with flag: {with_flag}, without flag: {without_flag}) +msg BADEXIT_OUT_OF_SYNC => Authorities disagree about the BadExit flag for {fingerprint} ({counts}) msg BANDWIDTH_AUTHORITIES_OUT_OF_SYNC => Bandwidth authorities have a substantially different number of measured entries: {authorities} msg AUTHORITY_UNAVAILABLE => Unable to retrieve the {fetch_type} from {authority} ({url}): {error} [View Less]

1 0

[doctor/master] Re-enable check that BadExit flags are in sync
by atagar＠torproject.org 21 Feb '16

21 Feb '16

commit 8ada4e0305b16f6b854dae839db2adb7446b7d37 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 17:58:05 2016 -0800 Re-enable check that BadExit flags are in sync Roger says these are useful to him and that he's willing to invest effort to keep in sync so re-enabling the checks... https://trac.torproject.org/projects/tor/ticket/17441#comment:3 --- consensus_health_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git … [View More]

1 0

[torspec/master] Update prop264 per discussion
by nickm＠torproject.org 21 Feb '16

21 Feb '16

commit 4b772b32b46eca2cc021b9aab7bde4bdfb1d8498 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sat Feb 20 20:32:50 2016 -0500 Update prop264 per discussion --- proposals/264-subprotocol-versions.txt | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/proposals/264-subprotocol-versions.txt b/proposals/264-subprotocol-versions.txt index 20174b9..7d84228 100644 --- a/proposals/264-subprotocol-versions.txt +++ b/proposals/264-subprotocol-… [View More]

1 0

[torspec/master] mark 265 accepted, targetting 029.
by nickm＠torproject.org 21 Feb '16

21 Feb '16

commit 1f07de13faaf997d4915e24837280f84a070c740 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sat Feb 20 20:17:23 2016 -0500 mark 265 accepted, targetting 029. --- proposals/000-index.txt | 4 ++-- proposals/265-load-balancing-with-overhead.txt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/proposals/000-index.txt b/proposals/000-index.txt index b5e3604..ae65d5a 100644 --- a/proposals/000-index.txt +++ b/proposals/000-index.… [View More]

1 0

[torspec/master] retitile 257 per discussion
by nickm＠torproject.org 21 Feb '16

21 Feb '16

commit 700c8a6f12cd7538844c21f629bdc74104a0d108 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sat Feb 20 20:16:02 2016 -0500 retitile 257 per discussion --- proposals/000-index.txt | 8 ++++---- proposals/257-hiding-authorities.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/proposals/000-index.txt b/proposals/000-index.txt index 20d6bb7..b5e3604 100644 --- a/proposals/000-index.txt +++ b/proposals/000-index.txt @@ -177,13 +177,13 @@ … [View More]Proposals by number: 254 Padding Negotiation [DRAFT] 255 Controller features to allow for load-balancing hidden services [DRAFT] 256 Key revocation for relays and authorities [OPEN] -257 Refactoring authorities and taking parts offline [META] +257 Refactoring authorities and making them more isolated from the net [META] 258 Denial-of-service resistance for directory authorities [ACCEPTED] 259 New Guard Selection Behaviour [DRAFT] 260 Rendezvous Single Onion Services [DRAFT] 261 AEZ for relay cryptography [OPEN] 262 Re-keying live circuits with new cryptographic material [OPEN] -263 Request to change key exchange protocol for handshake v1.1 [OPEN] +263 Request to change key exchange protocol for handshake v1.2 [OPEN] 264 Putting version numbers on the Tor subprotocols [OPEN] 265 Load Balancing with Overhead Parameters [DRAFT] 266 Removing current obsolete clients from the Tor network [DRAFT] @@ -243,7 +243,7 @@ Proposals by status: 256 Key revocation for relays and authorities 261 AEZ for relay cryptography 262 Re-keying live circuits with new cryptographic material - 263 Request to change key exchange protocol for handshake v1.1 + 263 Request to change key exchange protocol for handshake v1.2 264 Putting version numbers on the Tor subprotocols ACCEPTED: 140 Provide diffs between consensuses @@ -258,7 +258,7 @@ Proposals by status: 001 The Tor Proposal Process 098 Proposals that should be written 099 Miscellaneous proposals - 257 Refactoring authorities and taking parts offline + 257 Refactoring authorities and making them more isolated from the net FINISHED: 121 Hidden Service Authentication [in 0.2.1.x] 151 Improving Tor Path Selection diff --git a/proposals/257-hiding-authorities.txt b/proposals/257-hiding-authorities.txt index cab85ba..9c32f90 100644 --- a/proposals/257-hiding-authorities.txt +++ b/proposals/257-hiding-authorities.txt @@ -1,5 +1,5 @@ Filename: 257-hiding-authorities.txt -Title: Refactoring authorities and taking parts offline +Title: Refactoring authorities and making them more isolated from the net Authors: Nick Mathewson, Andrea Shepard Created: 2015-10-27 Status: Meta [View Less]

1 0

[doctor/master] Move TEST_RUN to util
by atagar＠torproject.org 20 Feb '16

20 Feb '16

commit 63eb5d36b8ee5f4b23d879482e1214e60c58e65a Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Feb 18 08:08:19 2016 -0800 Move TEST_RUN to util To test locally without spamming lists we have a simple TEST_RUN flag that prints results rather than emailing. Moving this into the util so all our scripts can take advantage of this. --- consensus_health_checker.py | 29 +++++++++-------------------- descriptor_checker.py | 2 +- … [View More]fingerprint_change_checker.py | 2 +- sybil_checker.py | 2 +- util.py | 9 +++++++++ 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/consensus_health_checker.py b/consensus_health_checker.py index 1962d10..e8de5bf 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2013, Damian Johnson and The Tor Project +# Copyright 2013-2016, Damian Johnson and The Tor Project # See LICENSE for licensing information """ @@ -23,10 +23,6 @@ import stem.util.enum from stem import Flag from stem.util.lru_cache import lru_cache -# Set this flag to print results rather than email them. - -TEST_RUN = False - Runlevel = stem.util.enum.UppercaseEnum('NOTICE', 'WARNING', 'ERROR') DIRECTORY_AUTHORITIES = stem.descriptor.remote.get_authorities() @@ -276,19 +272,16 @@ def main(): log.debug('Sending notification for issues (%s)' % ', '.join(destination_labels)) - if TEST_RUN: - print('\n'.join(map(str, issues))) - else: - body = '\n'.join(map(str, issues)) - cc = [d.address for d in destinations.values() if d and not d.bcc] - bcc = [d.address for d in destinations.values() if d and d.bcc] + body = '\n'.join(map(str, issues)) + cc = [d.address for d in destinations.values() if d and not d.bcc] + bcc = [d.address for d in destinations.values() if d and d.bcc] - util.send(EMAIL_SUBJECT, body = body, cc = cc, bcc = bcc) + util.send(EMAIL_SUBJECT, body = body, cc = cc, bcc = bcc) - # notification for #tor-bots + # notification for #tor-bots - body = '\n'.join(['[consensus-health] %s' % issue for issue in issues]) - util.send('Announce or', body = body, to = ['tor-misc(a)commit.noreply.org']) + body = '\n'.join(['[consensus-health] %s' % issue for issue in issues]) + util.send('Announce or', body = body, to = ['tor-misc(a)commit.noreply.org']) else: if issues: log.info("All %i issues were suppressed. Not sending a notification." % len(issues)) @@ -795,8 +788,4 @@ if __name__ == '__main__': except: msg = "consensus_health_checker.py failed with:\n\n%s" % traceback.format_exc() log.error(msg) - - if TEST_RUN: - print("Error: %s" % msg) - else: - util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS]) + util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS]) diff --git a/descriptor_checker.py b/descriptor_checker.py index 05a6513..953def1 100755 --- a/descriptor_checker.py +++ b/descriptor_checker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2013, Damian Johnson and The Tor Project +# Copyright 2013-2016, Damian Johnson and The Tor Project # See LICENSE for licensing information """ diff --git a/fingerprint_change_checker.py b/fingerprint_change_checker.py index c6a4b98..493f51c 100755 --- a/fingerprint_change_checker.py +++ b/fingerprint_change_checker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2015, Damian Johnson and The Tor Project +# Copyright 2015-2016, Damian Johnson and The Tor Project # See LICENSE for licensing information """ diff --git a/sybil_checker.py b/sybil_checker.py index 52fba0c..daaf0bd 100755 --- a/sybil_checker.py +++ b/sybil_checker.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2013, Damian Johnson and The Tor Project +# Copyright 2013-2016, Damian Johnson and The Tor Project # See LICENSE for licensing information """ diff --git a/util.py b/util.py index 9440d4c..483181e 100644 --- a/util.py +++ b/util.py @@ -15,6 +15,8 @@ FROM_ADDRESS = 'atagar(a)torproject.org' TO_ADDRESSES = ['tor-consensus-health(a)lists.torproject.org'] ERROR_ADDRESS = 'atagar(a)torproject.org' +TEST_RUN = False # print script results rather than emailing + def get_path(*comp): """ @@ -88,6 +90,13 @@ def send(subject, body, to = TO_ADDRESSES, cc = None, bcc = None): :raises: **Exception** if the email fails to be sent """ + if TEST_RUN: + print('Email to: %s' % to) + print('Subject: %s' % subject) + print('-' * 60) + print(body) + return + msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = FROM_ADDRESS [View Less]

1 0

[doctor/master] Support address ranges for relays we track
by atagar＠torproject.org 20 Feb '16

20 Feb '16

commit 44b3a702b34604d67cfaa0da50e5ae243e5c41c2 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 12:42:54 2016 -0800 Support address ranges for relays we track Ewww, but Stem's ExitPolicy already supports exactly what we want in a delightfully efficient way so taking advantage of that. --- track_relays.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/track_relays.py b/track_relays.py index a65356d..bc46216 100755 --- a/… [View More]

1 0

[doctor/master] Track when malicious relays return to the network
by atagar＠torproject.org 20 Feb '16

20 Feb '16

commit efa0ed43194bbc833ad91233586c40198070a94f Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 11:36:42 2016 -0800 Track when malicious relays return to the network When BadExit relays are removed we want to continue to monitor for their return for a time after. This is a script requested by David on... https://trac.torproject.org/projects/tor/ticket/18246 Basic script's done but as the TODO comments show there's still some … [View More]missing bits. --- data/tracked_relays.cfg | 22 +++++++ fingerprint_change_checker.py | 5 +- track_relays.py | 136 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 4 deletions(-) diff --git a/data/tracked_relays.cfg b/data/tracked_relays.cfg new file mode 100644 index 0000000..e25959e --- /dev/null +++ b/data/tracked_relays.cfg @@ -0,0 +1,22 @@ +# Relays we want notifications for when they reappear. This can be done by +# address (single addresses or ranges) or fingerprints. +# +# Anything in this file is public. All entries *MUST* have a description and an +# expiration date for when it can be removed. The description is particularly +# important so we can puzzle out why we were monitoring for this if it +# reappears. +# +# Few examples... +# +# PrivacyPT.description running sslstrip on 2011-01-05 +# PrivacyPT.expires 2016-01-15 +# PrivacyPT.address 84.90.72.186 +# +# trotsky.description sybil consisting of 383 exit relays on 2010-10-02 +# trotsky.expires 2016-02-01 +# trotsky.address 185.19.80.0/26 +# +# Unnamed001.description harvesting hidden services on 2012-08-12 +# Unnamed001.expires 2016-03-01 +# Unnamed001.fingerprint 05AF83344B3787D0DCCD47DC4A6A4668142A5F8C + diff --git a/fingerprint_change_checker.py b/fingerprint_change_checker.py index 493f51c..e13f3d0 100755 --- a/fingerprint_change_checker.py +++ b/fingerprint_change_checker.py @@ -89,10 +89,7 @@ def main(): body += "\n" - try: - util.send(EMAIL_SUBJECT, body = body, to = ['bad-relays(a)lists.torproject.org', 'atagar(a)torproject.org']) - except Exception as exc: - log.warn("Unable to send email: %s" % exc) + util.send(EMAIL_SUBJECT, body = body, to = ['bad-relays(a)lists.torproject.org', 'atagar(a)torproject.org']) # register that we've notified for these diff --git a/track_relays.py b/track_relays.py new file mode 100755 index 0000000..e770006 --- /dev/null +++ b/track_relays.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# Copyright 2016, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Notifies if specific relays reappear in the network. +""" + +import datetime +import traceback + +import stem.descriptor.remote +import stem.util.conf + +import util + +log = util.get_logger('track_relays') + +EMAIL_SUBJECT = 'Relays Returned' + +EMAIL_BODY = """\ +The following previously BadExit relays have returned to the network... + +""" + + +class TrackedRelay(object): + """ + Represents a relay we're keeping an eye on. + + :var str identifier: brief identifier given to the entry + :var str description: description of why we're tracking it + :var datetime expires: when this entry expires + :var str address: address of the relay we're tracking + :var str fingerprint: fingerprint of the relay we're tracking + """ + + def __init__(self, identifier, config): + self.identifier = identifier + self.description = config.get('%s.description' % identifier, '') + + expires_str = config.get('%s.expires' % identifier, '') + + if not expires_str: + raise ValueError("Our config file is missing a '%s.expires' entry" % identifier) + + try: + self.expires = datetime.datetime.strptime(expires_str, '%Y-%m-%d') + except ValueError: + raise ValueError("'%s.expires' is malformed. We expect it to be in the form 'Year-Month-Day'" % identifier) + + self.address = config.get('%s.address' % identifier, None) + self.fingerprint = config.get('%s.fingerprint' % identifier, None) + + if not self.address and not self.fingerprint: + raise ValueError("We need either a '%s.address' or '%s.fingerprint' to track" % (identifier, identifier)) + + def __str__(self): + attr = [] + + if self.address: + attr.append('address: %s' % self.address) + + if self.fingerprint: + attr.append('fingerprint: %s' % self.fingerprint) + + return '%s (%s)' % (self.identifier, ', '.join(attr)) + + +def get_tracked_relays(): + """ + Provides the relays we're tracking. + + :returns: **list** of **TrackedRelay** we're tracking + + :raises: **ValueError** if our config file is malformed + """ + + config = stem.util.conf.get_config('tracked_relays') + config.load(util.get_path('data', 'tracked_relays.cfg')) + + # TODO: check for expired entries + + identifiers = set([key.split('.')[0] for key in config.keys()]) + return [TrackedRelay(identifier, config) for identifier in identifiers] + + +def main(): + # Map addresses and fingerprints to relays for constant time lookups. Address + # ranges are handled separately cuz... well, they're a pita. + + tracked_addresses = {} + tracked_address_ranges = {} + tracked_fingerprints = {} + + for relay in get_tracked_relays(): + if relay.address: + if '/' in relay.address: + tracked_address_ranges[relay.address] = relay + else: + tracked_addresses[relay.address] = relay + + if relay.fingerprint: + tracked_fingerprints[relay.fingerprint] = relay + + downloader = stem.descriptor.remote.DescriptorDownloader() + found_relays = {} # mapping of TrackedRelay => RouterStatusEntry + + for desc in downloader.get_consensus(): + if desc.address in tracked_addresses: + found_relays[tracked_addresses[desc.address]] = desc + elif desc.fingerprint in tracked_fingerprints: + found_relays[tracked_fingerprints[desc.fingerprint]] = desc + else: + pass # TODO: implement for tracked_address_ranges + + if found_relays: + log.debug("Sending a notification for %i relay entries..." % len(found_relays)) + body = EMAIL_BODY + + for tracked_relay, desc in found_relays.items(): + log.debug('* %s' % tracked_relay) + body += '* %s (%s)\n' % (tracked_relay.identifier, tracked_relay.description) + body += ' address: %s\n' % desc.address + body += ' fingerprint: %s\n\n' % desc.fingerprint + + util.send(EMAIL_SUBJECT, body = body, to = ['bad-relays(a)lists.torproject.org', 'atagar(a)torproject.org']) + + +if __name__ == '__main__': + try: + main() + except: + msg = "track_relays.py failed with:\n\n%s" % traceback.format_exc() + log.error(msg) + util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS]) [View Less]

1 0

[doctor/master] Notify for expired tracked_relays.cfg entries
by atagar＠torproject.org 20 Feb '16

20 Feb '16

commit dc84ea9cf3992b16f938dfb007e153dc85a77392 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 20 12:20:43 2016 -0800 Notify for expired tracked_relays.cfg entries Proper expiring of tracked_relays.cfg entries. When the date passes those entries are ignored and I'm notififed. --- track_relays.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/track_relays.py b/track_relays.py index e770006..a65356d 100755 ---… [View More]

1 0