- tor-commits - lists.torproject.org

[torbel/master] Fix bulklist error when no ip is given
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 1097384c848d4fbfc2539df4b15df1bc7c7d98d6 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 09:56:44 2011 +0200 Fix bulklist error when no ip is given --- bulklist.fcgi | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index 5b1913a..1e28f01 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -49,7 +49,7 @@ def app(environ, start_response): if ip != "": try: - ip = query.ip_from_string(ip) + ip = query.ip_from_string(ip) except socket.error: ip = 0 @@ -85,7 +85,7 @@ def app(environ, start_response): yield '<body>\n' yield '<center>\n' yield '\n' - yield '<br>\n'); + yield '<br>\n' yield '\n'

1 0

[torbel/master] Steal most of the bulklist text from the old script
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 2de22deb9d76f99bd15c239957b6b538251044fc Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 09:52:35 2011 +0200 Steal most of the bulklist text from the old script --- bulklist.fcgi | 38 ++++++++++++++++++++++++++++++++++++-- 1 files changed, 36 insertions(+), 2 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index 92dfcff..5b1913a 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -72,8 +72,42 @@ def app(environ, start_response): yield '%s\n' % IPAddress(i) else: start_response('200 OK', [('Content-Type', 'text/plain')]) - yield 'bai' - + yield '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" ' + yield '"http://www.w3.org/TR/REC-html40/loose.dtd">\n' + yield '<html>\n' + yield '<head>\n' + yield '<meta http-equiv="content-type" content="text/html; ' + yield 'charset=utf-8">\n' + yield '<title>Bulk Tor Exit Exporter</title>\n' + yield '<link rel="shortcut icon" type="image/x-icon" ' + yield 'href="./favicon.ico">\n' + yield '</head>\n' + yield '<body>\n' + yield '<center>\n' + yield '\n' + yield '<br>\n'); + + yield '\n' + + yield 'Welcome to the Tor Bulk Exit List exporting tool.<br><br>\n' + yield 'If you are a service provider and you wish to build a list ' + yield 'of possible Tor nodes that might contact one of your servers, ' + yield 'enter that single server address below. Giving you the whole ' + yield 'list means you can query the list privately, rather than ' + yield 'telling us your users\' IP addresses.\n' + yield 'This list allows you to have a nearly real time authoritative ' + yield 'source for Tor exits that allow contacting your server on ' + yield 'port 80 or the given port.<br><br>\n' + + yield 'Please enter an IP address:<br>\n' + yield '<form action="/cgi-bin/TorBulkExitList.py" name="ip">\n' + yield '<input type="text" name="ip"><br>\n' + yield '<input type="submit" value="Submit">' + yield '</form>' + + yield '</center>\n' + yield '</body>' + yield '</html>' if __name__ == "__main__": WSGIServer(app).run()

1 0

[torbel/master] Implement narrow port test and recording.
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 3b7eb46407e934ffb6f5d7a77304815017fb0ed4 Author: Harry Bock <hbock(a)ele.uri.edu> Date: Tue Sep 7 17:42:32 2010 -0400 Implement narrow port test and recording. --- controller.py | 42 ++++++++++++++++++++++++++++++++++-------- 1 files changed, 34 insertions(+), 8 deletions(-) diff --git a/controller.py b/controller.py index 3a1258e..29beaff 100644 --- a/controller.py +++ b/controller.py @@ -16,7 +16,7 @@ from datetime import datetime, timedelta if sys.version_info >= (2,6): import json - + # TODO: Choose the best reactor for the platform. from twisted.internet import epollreactor epollreactor.install() @@ -233,6 +233,9 @@ class Controller(TorCtl.EventHandler): if not config.test_host: config.test_host = conn.get_info("address")["address"] + # Resolve config.test_host for later use. + self.test_ip = socket.gethostbyname(config.test_host) + log.notice("Our external test IP address should be %s.", config.test_host) # Build a list of Guard routers, so we have a list of reliable @@ -259,7 +262,7 @@ class Controller(TorCtl.EventHandler): router, controller = self) reactor.connectTCP(config.tor_host, config.tor_port, f) return f.connectDeferred - + for port in router.current_test.test_ports: # Initiate bookkeeping for this stream, tracking it # by source port, useful when we only have a socket as reference. @@ -278,7 +281,7 @@ class Controller(TorCtl.EventHandler): def closeCallback(sport): self.stream_remove(source_port = sport) - + connect = socksConnect(router, port) connect.addCallback(connectCallback) @@ -344,6 +347,14 @@ class Controller(TorCtl.EventHandler): if router.current_test.is_complete(): self.end_test(router) + def narrow(self, router, port): + """ Mark port as working for router's current test, and end the test if + it is complete. """ + if router.current_test: + router.current_test.narrow(port) + if router.current_test.is_complete(): + self.end_test(router) + def failed(self, router, port): """ Mark port as failed for router's current test, and end the test if it is complete. """ @@ -788,10 +799,22 @@ class Controller(TorCtl.EventHandler): router = stream.router if event.reason == "END": - fail = True + fail = True + narrow = False # The router is not allowing traffic to this port currently. if event.remote_reason == "EXITPOLICY": - fail = True + # If this is due to it being a narrow exit, mark it as + # such. + if router.is_narrow_exit(self.test_ip, event.target_port): + fail = False + narrow = True + log.debug("Narrow exit: %s %s", + router.nickname, + router.exit_policy_string()) + # Otherwise, this is a general exit policy failure. + else: + fail = True + # This router is hibernating and will not allow traffic. elif event.remote_reason == "HIBERNATING": # TODO: Mark as inaccessible. @@ -831,10 +854,13 @@ class Controller(TorCtl.EventHandler): # Bail if we closed. return - if fail: - self.failed(stream.router, event.target_port) + if narrow: + self.narrow(stream.router, event.target_port) else: - self.passed(stream.router, event.target_port) + if fail: + self.failed(stream.router, event.target_port) + else: + self.passed(stream.router, event.target_port) return

1 0

[torbel/master] query.ExitList expects filename argument
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 1b61735a60a2e23e84609accac2931c48b5775cd Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Fri Sep 2 05:26:03 2011 +0200 query.ExitList expects filename argument --- checkweb.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/checkweb.py b/checkweb.py index c3d90ab..1d7d799 100755 --- a/checkweb.py +++ b/checkweb.py @@ -5,7 +5,7 @@ from cgi import escape from logger import * from flup.server.fcgi import WSGIServer -elist = query.ExitList(csv_file = "bel_export.csv") +elist = query.ExitList(filename = "bel_export.csv") port = 8000 log = get_logger("WSGITest")

1 0

[torbel/master] Fix bug in torbel.query gzip import support.
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 2989b4f608c029523944fa3697c302f56547b642 Author: Harry Bock <hbock(a)ele.uri.edu> Date: Sat Sep 18 21:14:50 2010 -0400 Fix bug in torbel.query gzip import support. Missing a negative sign in the index. --- query.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/query.py b/query.py index f1d0046..69e9ec9 100644 --- a/query.py +++ b/query.py @@ -265,7 +265,7 @@ class ExitList: gzipped. """ if filename.endswith(".gz"): infile = gzip.open(filename, "rb") - filename = filename[:3] + filename = filename[:-3] else: infile = open(filename, "rb")

1 0

[torbel/master] Add the bulk exit list tool
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit affe14eee229adb36b4bef5c5de5afe7e6da6b1b Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 07:58:04 2011 +0200 Add the bulk exit list tool --- bulklist.fcgi | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 78 insertions(+), 0 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi new file mode 100755 index 0000000..56dee21 --- /dev/null +++ b/bulklist.fcgi @@ -0,0 +1,78 @@ +#!/usr/bin/python +import sys, os + +sys.path.append('/home/torbel') +sys.path.append('/home/torbel/torflow') +sys.path.append('/home/torbel/torbel') + +import socket + +import torbel.query as query +from cgi import escape +from urlparse import parse_qs +from logger import * +from flup.server.fcgi import WSGIServer + +import time +from threading import Timer + +import math + +from ipaddr import IPAddress + +elist = None + +log = get_logger("BulkExitList") + +def update_elist(): + global elist + elist = query.ExitList(filename = "/home/torbel/export/torbel_export.csv", + status_filename = "/home/torbel/export/torbel_export.status") + nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) + if nextUpdate > 0: + log.debug("Scheduling update in %d seconds.", nextUpdate) + #Timer(nextUpdate, update_elist, ()).start() + else: + log.notice("Export file is not up-to-date. Trying again in 10 minutes.") + #Timer(10*60, update_elist, ()).start() + +update_elist() + +def app(environ, start_response): + q = parse_qs(environ['QUERY_STRING']) + + + ip = escape(q.get('ip', [''])[0]) + port = escape(q.get('port', [''])[0]) + + if ip != "": + try: + ip = query.ip_from_string(ip) + except socket.error: + ip = 0 + + if ip != 0: + try: + port = int(port) + if port > 65535 or port < 1: + port = 80 + except ValueError: + port = 80 + exits = elist.will_exit_to(ip, port) + + start_response('200 OK', [('Content-Type', 'text/plain')]) + + yield "# This is a list of all Tor exit nodes that can contact %s " % IPAddress(ip) + yield "on Port %d #\n" % port + yield "# This file was generated on %s UTC #\n" % time.asctime(time.gmtime()) + exits.sort() + for i in exits: + yield '%s\n' % IPAddress(i) + else: + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield 'bai' + + +if __name__ == "__main__": + WSGIServer(app).run() +

1 0

[torbel/master] Add some new requirements to the README
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 0b8aaf185c5ee990341df21467007a207973eaaf Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Thu Jul 21 17:45:45 2011 +0200 Add some new requirements to the README --- README | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README b/README index 5e2fa87..eb618ab 100644 --- a/README +++ b/README @@ -20,7 +20,7 @@ Software Requirements The following are required to run TorBEL on your system: - - Tor 0.2.2.13-alpha or higher recommended, with an open + - Tor 0.2.2.30-rc or higher recommended, with an open control port. - Python (>= 2.5 is required, >= 2.6 is recommended and required for JSON support) @@ -52,6 +52,7 @@ is available in your torrc: - FetchDirInfoExtraEarly 1 (Tor >= 0.2.2.13-alpha) - LearnCircuitBuiltTimeout 0 (Tor >= 0.2.2.14-alpha) - WarnUnsafeSocks 0 (Tor >= 0.2.2.14-alpha) + - FetchV2Networkstatus 1 (Tor >= 0.2.2.26-beta) These options significantly affect Tor's operation and thus you should not run other applications that depend on Tor's normal operation with the same instance used by TorBEL. @@ -151,4 +152,4 @@ You can configure TorBEL to log to any combination of stdout, syslog, and a (non-rotated) log file. Logging to file is extremely useful for debugging. Be warned that if your log_level is set to DEBUG or lower, your daemon.log (or wherever your syslog ends up) will get utterly -spammed. You have been warned! \ No newline at end of file +spammed. You have been warned!

1 0

[torbel/master] Add metadata to CSV export format.
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 8323b633dbccba0b3e06362cc8961a6f303557d8 Author: Harry Bock <hbock(a)ele.uri.edu> Date: Wed Oct 20 00:43:42 2010 -0400 Add metadata to CSV export format. Add metadata to CSV format specification and implementation (query and export). This allows torbel to know what version of the CSV format to use, allowing for changes in the future and backward compatibility. Not implemented yet for the JSON format. --- __init__.py | 2 ++ controller.py | 4 ++-- doc/data-spec.txt | 13 +++++++++++++ query.py | 24 ++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index e69de29..b5bf7e9 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,2 @@ +# TorBEL export version format. +__export_version__ = 1 diff --git a/controller.py b/controller.py index 29beaff..807ce18 100644 --- a/controller.py +++ b/controller.py @@ -24,7 +24,7 @@ from twisted.internet import reactor from TorCtl import TorCtl, TorUtil # torbel submodules -from torbel import scheduler, network, utils +from torbel import scheduler, network, utils, __export_version__ from torbel.logger import * from torbel.router import RouterRecord @@ -598,7 +598,7 @@ class Controller(TorCtl.EventHandler): csv_file = open(fn_new, "w") out = csv.writer(csv_file, dialect = csv.excel) - + out.writerow(["torbel", __export_version__]) # FIXME: Is it safe to just take the itervalues list? with self.consensus_cache_lock: for router in self.router_cache.itervalues(): diff --git a/doc/data-spec.txt b/doc/data-spec.txt index c5f4014..9c39d87 100644 --- a/doc/data-spec.txt +++ b/doc/data-spec.txt @@ -12,6 +12,8 @@ Status: Draft This document is a work-in-progress and the data format may change quickly during the summer in response to demand and implementation problems. + This document describes TorBEL export data format version 1. + 1. Exported Data 1.1. Records @@ -152,6 +154,17 @@ Status: Draft Fields that contain an escaped double quote are also enclosed in a set of double quotes. + Metadata indicating the CSV export format version must be present on the first + line in the format: + + torbel,VERSION + + Where VERSION is the export format version as an integer, and torbel is + the literal string 'torbel'. Additional metadata fields may be added + in future export versions and consumers MUST handle extraneous fields + on this line not specified in this document. + + Following the metadata line, all remaining lines are router data rows. The fields of each row are, in order: ExitAddress, RouterID, Nickname, LastTestedTimestamp, InConsensus, diff --git a/query.py b/query.py index 69e9ec9..16a6e7a 100644 --- a/query.py +++ b/query.py @@ -10,6 +10,7 @@ import ipaddr import sys from socket import inet_aton, inet_ntoa from logger import * +from torbel import __export_version__ if sys.version_info >= (2,6): import json @@ -189,10 +190,14 @@ class ExitPolicyRule: return "reject " + ip + ":" + port class ExitList: + class ImportError(ValueError): + pass + def __init__(self, filename, status_filename = None): self.cache_ip = {} self.cache_id = {} + self.version = None self.next_update = None self.last_update = None self.export_files = [] @@ -286,6 +291,25 @@ class ExitList: the TorBEL data-spec document. """ reader = csv.reader(infile, dialect = "excel") record = 1 + # Grab metadata row and export format version. + metadata = reader.next() + try: + self.version = int(metadata[1]) + if metadata[0] != "torbel": + raise self.ImportError("Invalid TorBEL export format.") + if self.version > __export_version__: + raise self.ImportError("Export version %d not supported!" % self.version) + + # ValueError will be raised if the first value on the metadata line + # is not an integer. + # IndexError is raised if the metadata line is empty. Not quite sure + # if this is actually possible! + # StopIteration is raised if we try to read from an empty file. + # All of these indicate the TorBEL export file is not actually + # a valid export. + except (ValueError, IndexError, StopIteration): + raise self.ImportError("Invalid TorBEL export format.") + for r in reader: try: data = {

1 0

[torbel/master] Use UTC consistently
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit cbc5d8c69e9386b15a3d366a1b3e9a4b379c96ba Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 08:32:34 2011 +0200 Use UTC consistently --- bulklist.fcgi | 4 +++- checkweb.py | 3 ++- controller.py | 2 +- dnsel/dnsel.py | 5 +++-- doc/data-spec.txt | 6 +++--- query.py | 6 +++--- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index c447903..92dfcff 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -7,6 +7,8 @@ sys.path.append('/home/torbel/torbel') import socket +from calendar import timegm + import torbel.query as query from cgi import escape from urlparse import parse_qs @@ -28,7 +30,7 @@ def update_elist(): global elist elist = query.ExitList(filename = "/home/torbel/export/torbel_export.csv", status_filename = "/home/torbel/export/torbel_export.status") - nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) + nextUpdate = math.ceil(timegm(elist.next_update.timetuple()) - time.time()) if nextUpdate > 0: log.debug("Scheduling update in %d seconds.", nextUpdate) Timer(nextUpdate, update_elist, ()).start() diff --git a/checkweb.py b/checkweb.py index 85a99a5..32c6388 100755 --- a/checkweb.py +++ b/checkweb.py @@ -11,6 +11,7 @@ from logger import * from flup.server.fcgi import WSGIServer import time +from calendar import timegm from threading import Timer import math @@ -25,7 +26,7 @@ def update_elist(): global elist elist = query.ExitList(filename = "/home/torbel/export/torbel_export.csv", status_filename = "/home/torbel/export/torbel_export.status") - nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) + nextUpdate = math.ceil(timegm(elist.next_update.timetuple()) - time.time()) if nextUpdate > 0: log.debug("Scheduling update in %d seconds.", nextUpdate) Timer(nextUpdate, update_elist, ()).start() diff --git a/controller.py b/controller.py index 807ce18..fe6dbf7 100644 --- a/controller.py +++ b/controller.py @@ -524,7 +524,7 @@ class Controller(TorCtl.EventHandler): # Add 30 seconds to the config.export_interval time so clients don't # try to fetch it as we're writing it. - next = datetime.now() + timedelta(0, config.export_interval * 60 + 30) + next = datetime.utcnow() + timedelta(0, config.export_interval * 60 + 30) nextstr = next.strftime("%b %d %Y %H:%M:%S") export_file_prefix = config.export_file_prefix fn = export_file_prefix + ".status" diff --git a/dnsel/dnsel.py b/dnsel/dnsel.py index 2060562..b39230a 100755 --- a/dnsel/dnsel.py +++ b/dnsel/dnsel.py @@ -6,6 +6,7 @@ # and the TorBEL query interface. import os, sys, time from datetime import datetime +from calendar import timegm from zope.interface import implements from twisted.names import server, dns, common, error @@ -32,7 +33,7 @@ class TorDNSServerFactory(server.DNSServerFactory): log.info("Export %s likely stale.", filename) # Set up updates. - nextUpdate = time.mktime(self.el.next_update.timetuple()) - time.time() + nextUpdate = timegm(self.el.next_update.timetuple()) - time.time() if nextUpdate > 0: log.debug("Scheduling first update in %.1f seconds.", nextUpdate) reactor.callLater(nextUpdate, self.update) @@ -45,7 +46,7 @@ class TorDNSServerFactory(server.DNSServerFactory): def update(self): next = self.el.update() - nextUpdate = time.mktime(next.timetuple()) - time.time() + nextUpdate = timegm(next.timetuple()) - time.time() if nextUpdate > 0: log.info("ExitList updated. Next update in %.1f seconds.", nextUpdate) reactor.callLater(nextUpdate, self.update) diff --git a/doc/data-spec.txt b/doc/data-spec.txt index 9c39d87..5d6db11 100644 --- a/doc/data-spec.txt +++ b/doc/data-spec.txt @@ -108,9 +108,9 @@ Status: Draft * NextUpdate "Datetime": The value of this key is a date-time string with strftime(3) format - "%b %d %Y %H:%M:%S". This date represents the absolute earliest time at - which a TorBEL export consumer should attempt to fetch an updated export - file. + "%b %d %Y %H:%M:%S" in UTC. This date represents the absolute earliest + time at which a TorBEL export consumer should attempt to fetch an + updated export file. An example of this key is: NextUpdate "Aug 12 16:51:04" This key must appear exactly once in the status file. diff --git a/query.py b/query.py index 16a6e7a..a5dda85 100644 --- a/query.py +++ b/query.py @@ -212,7 +212,7 @@ class ExitList: def should_update(self): """ Returns True if our updates are out of date. """ - return self.next_update and (datetime.datetime.now() > self.next_update) + return self.next_update and (datetime.datetime.utcnow() > self.next_update) def update(self, force = False): """ Read and update to the latest export and status files. @@ -232,13 +232,13 @@ class ExitList: # Import the status file, if available. if self.status_filename: self.next_update, self.export_files = self.read_status(self.status_filename) - self.stale = (datetime.datetime.now() > self.next_update) + self.stale = (datetime.datetime.utcnow() > self.next_update) else: self.next_update = None self.export_files = None # Record our last update time. - self.last_update = datetime.datetime.now() + self.last_update = datetime.datetime.utcnow() return self.next_update

1 0

[torbel/master] Actually start the timer for the refresh
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 643548bf399b363e630ba4c62e94c26517a05aec Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 07:58:36 2011 +0200 Actually start the timer for the refresh --- bulklist.fcgi | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index 56dee21..c447903 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -31,10 +31,10 @@ def update_elist(): nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) if nextUpdate > 0: log.debug("Scheduling update in %d seconds.", nextUpdate) - #Timer(nextUpdate, update_elist, ()).start() + Timer(nextUpdate, update_elist, ()).start() else: log.notice("Export file is not up-to-date. Trying again in 10 minutes.") - #Timer(10*60, update_elist, ()).start() + Timer(10*60, update_elist, ()).start() update_elist()

1 0