commit e9047744d7f25e807737574b7e87140cb109f187 Author: Damian Johnson atagar@torproject.org Date: Sat May 28 16:43:59 2016 -0700
Report unreachable fallback directories
New notifications requested by teor that provides a summary if more than 25% of the fallback directories are unreachable or slow. Notices are sent to teor, nickm, me, and #tor-bots...
https://trac.torproject.org/projects/tor/ticket/18177 --- consensus_health_checker.py | 11 ++---- fallback_directories.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ util.py | 28 +++++++++++++++ 3 files changed, 114 insertions(+), 8 deletions(-)
diff --git a/consensus_health_checker.py b/consensus_health_checker.py index c1aed79..a34e972 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -9,7 +9,6 @@ Performs a variety of checks against the present votes and consensus. import collections import datetime import os -import socket import time import traceback
@@ -735,14 +734,10 @@ def is_orport_reachable(latest_consensus, consensuses, votes): continue # authority isn't in the consensus
for address, port, is_ipv6 in desc.or_addresses: - orport_socket = socket.socket(socket.AF_INET6 if is_ipv6 else socket.AF_INET, socket.SOCK_STREAM) + issue = util.check_reachability(address, port)
- try: - orport_socket.connect((address, port)) - except Exception as exc: - issues.append(Issue(Runlevel.WARNING, 'UNABLE_TO_REACH_ORPORT', authority = authority.nickname, address = address, port = port, error = exc, to = [authority])) - finally: - orport_socket.close() + if issue: + issues.append(Issue(Runlevel.WARNING, 'UNABLE_TO_REACH_ORPORT', authority = authority.nickname, address = address, port = port, error = issue, to = [authority]))
return issues
diff --git a/fallback_directories.py b/fallback_directories.py new file mode 100755 index 0000000..b9d6cc1 --- /dev/null +++ b/fallback_directories.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# Copyright 2016, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Report for how many of our fallback directories are unreachable. +""" + +import time +import traceback + +from stem.descriptor.remote import DescriptorDownloader, FallbackDirectory + +import util + +log = util.get_logger('fallback_directories') + +NOTIFICATION_THRESHOLD = 25 # send notice if this percentage of fallbacks are unusable +TO_ADDRESSES = ['atagar@torproject.org', 'teor2345@gmail.com', 'nickm@torproject.org'] +EMAIL_SUBJECT = 'Fallback Directory Summary' + +EMAIL_BODY = """\ +%i/%i (%i%%) fallback directories have become slow or unresponsive... + +""" + +downloader = DescriptorDownloader(timeout = 30) + + +def main(): + try: + fallback_directories = FallbackDirectory.from_remote().values() + log.info('Retrieved %i fallback directories' % len(fallback_directories)) + except IOError as exc: + raise IOError("Unable to determine tor's fallback directories: %s" % exc) + + issues = [] + + for relay in fallback_directories: + if not util.is_reachable(relay.address, relay.or_port): + log.info('%s ORPort unreachable' % relay.fingerprint) + issues.append('%s => ORPort is unreachable (%s:%i)' % (relay.fingerprint, relay.address, relay.or_port)) + continue + + if not util.is_reachable(relay.address, relay.dir_port): + log.info('%s DirPort unreachable' % relay.fingerprint) + issues.append('%s => DirPort is unreachable (%s:%i)' % (relay.fingerprint, relay.address, relay.dir_port)) + continue + + if relay.orport_v6 and not util.is_reachable(relay.orport_v6[0], relay.orport_v6[1]): + log.info('%s IPv6 ORPort unreachable' % relay.fingerprint) + issues.append('%s => IPv6 ORPort is unreachable (%s:%i)' % (relay.fingerprint, relay.orport_v6[0], relay.orport_v6[1])) + continue + + start = time.time() + downloader.get_consensus(endpoints = [(relay.address, relay.dir_port)]).run() + download_time = time.time() - start + log.info('%s download time was %0.1f seconds' % (relay.fingerprint, download_time)) + + if download_time > 15: + issues.append('%s => Downloading the consensus took %0.1f seconds' % (relay.fingerprint, download_time)) + + issue_percent = 100.0 * len(issues) / len(fallback_directories) + log.info('%i ssues found (%i%%)' % (len(issues), issue_percent)) + + if issue_percent >= NOTIFICATION_THRESHOLD: + log.info('Sending notification') + body = EMAIL_BODY % (len(issues), len(fallback_directories), 100.0 * len(issues) / len(fallback_directories)) + util.send(EMAIL_SUBJECT, body = body + '\n'.join([' * %s' % issue for issue in issues]), to = TO_ADDRESSES) + + # notification for #tor-bots + + body = '\n'.join(['[fallback-directories] %s' % issue for issue in issues]) + util.send('Announce or', body = body, to = ['tor-misc@commit.noreply.org']) + + +if __name__ == '__main__': + try: + main() + except: + msg = "fallback_directories.py failed with:\n\n%s" % traceback.format_exc() + log.error(msg) + util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS]) diff --git a/util.py b/util.py index 483181e..9d4adcb 100644 --- a/util.py +++ b/util.py @@ -4,11 +4,13 @@ Module for issuing email notifications to me via gmail.
import logging import os +import socket import smtplib
from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText
+import stem.util.connection import stem.util.log
FROM_ADDRESS = 'atagar@torproject.org' @@ -55,6 +57,32 @@ def get_logger(name): return log
+def is_reachable(address, port): + return check_reachability(address, port) == None + + +def check_reachability(address, port): + """ + Simple check to see if we can establish a connection to the given endpoint. + + :param str address: IPv4 or IPv6 address to check + :param int port: port to check + + :returns: **None** if the endpoint is reachable and a **str** describing the issue otherwise + """ + + socket_type = socket.AF_INET6 if stem.util.connection.is_valid_ipv6_address(address) else socket.AF_INET + test_socket = socket.socket(socket_type, socket.SOCK_STREAM) + + try: + test_socket.connect((address, port)) + return None + except Exception as exc: + return str(exc) + finally: + test_socket.close() + + def log_stem_debugging(name): """ Logs trace level stem output to the given log file.