tor-commits September 2011

tor-commits@lists.torproject.org

19 participants
865 discussions

[torbel/master] Allow users to enter a port
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit b16ed8464acb8eb0463cde10c00066fc9e5ec322 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 10:05:14 2011 +0200 Allow users to enter a port --- bulklist.fcgi | 15 ++++++++------- 1 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index 1e28f01..ccd96b4 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -44,14 +44,13 @@ def app(environ, start_response): q = parse_qs(environ['QUERY_STRING']) - ip = escape(q.get('ip', [''])[0]) + orig_ip = escape(q.get('ip', [''])[0]) port = escape(q.get('port', [''])[0]) - if ip != "": - try: - ip = query.ip_from_string(ip) - except socket.error: - ip = 0 + try: + ip = query.ip_from_string(orig_ip) + except socket.error: + ip = 0 if ip != 0: try: @@ -71,7 +70,7 @@ def app(environ, start_response): for i in exits: yield '%s\n' % IPAddress(i) else: - start_response('200 OK', [('Content-Type', 'text/plain')]) + start_response('200 OK', [('Content-Type', 'text/html')]) yield '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" ' yield '"http://www.w3.org/TR/REC-html40/loose.dtd">\n' yield '<html>\n' @@ -102,6 +101,8 @@ def app(environ, start_response): yield 'Please enter an IP address:<br>\n' yield '<form action="/cgi-bin/TorBulkExitList.py" name="ip">\n' yield '<input type="text" name="ip"><br>\n' + yield 'and optionally a Port:<br>\n' + yield '<input type="text" name="port"><br>\n' yield '<input type="submit" value="Submit">' yield '</form>'

1 0

[torbel/master] Fix bulklist error when no ip is given
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 1097384c848d4fbfc2539df4b15df1bc7c7d98d6 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 09:56:44 2011 +0200 Fix bulklist error when no ip is given --- bulklist.fcgi | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index 5b1913a..1e28f01 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -49,7 +49,7 @@ def app(environ, start_response): if ip != "": try: - ip = query.ip_from_string(ip) + ip = query.ip_from_string(ip) except socket.error: ip = 0 @@ -85,7 +85,7 @@ def app(environ, start_response): yield '<body>\n' yield '<center>\n' yield '\n' - yield '<br>\n'); + yield '<br>\n' yield '\n'

1 0

[torbel/master] Steal most of the bulklist text from the old script
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 2de22deb9d76f99bd15c239957b6b538251044fc Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 09:52:35 2011 +0200 Steal most of the bulklist text from the old script --- bulklist.fcgi | 38 ++++++++++++++++++++++++++++++++++++-- 1 files changed, 36 insertions(+), 2 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index 92dfcff..5b1913a 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -72,8 +72,42 @@ def app(environ, start_response): yield '%s\n' % IPAddress(i) else: start_response('200 OK', [('Content-Type', 'text/plain')]) - yield 'bai' - + yield '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" ' + yield '"http://www.w3.org/TR/REC-html40/loose.dtd">\n' + yield '<html>\n' + yield '<head>\n' + yield '<meta http-equiv="content-type" content="text/html; ' + yield 'charset=utf-8">\n' + yield '<title>Bulk Tor Exit Exporter</title>\n' + yield '<link rel="shortcut icon" type="image/x-icon" ' + yield 'href="./favicon.ico">\n' + yield '</head>\n' + yield '<body>\n' + yield '<center>\n' + yield '\n' + yield '<br>\n'); + + yield '\n' + + yield 'Welcome to the Tor Bulk Exit List exporting tool.<br><br>\n' + yield 'If you are a service provider and you wish to build a list ' + yield 'of possible Tor nodes that might contact one of your servers, ' + yield 'enter that single server address below. Giving you the whole ' + yield 'list means you can query the list privately, rather than ' + yield 'telling us your users\' IP addresses.\n' + yield 'This list allows you to have a nearly real time authoritative ' + yield 'source for Tor exits that allow contacting your server on ' + yield 'port 80 or the given port.<br><br>\n' + + yield 'Please enter an IP address:<br>\n' + yield '<form action="/cgi-bin/TorBulkExitList.py" name="ip">\n' + yield '<input type="text" name="ip"><br>\n' + yield '<input type="submit" value="Submit">' + yield '</form>' + + yield '</center>\n' + yield '</body>' + yield '</html>' if __name__ == "__main__": WSGIServer(app).run()

1 0

[torbel/master] query.ExitList expects filename argument
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 1b61735a60a2e23e84609accac2931c48b5775cd Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Fri Sep 2 05:26:03 2011 +0200 query.ExitList expects filename argument --- checkweb.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/checkweb.py b/checkweb.py index c3d90ab..1d7d799 100755 --- a/checkweb.py +++ b/checkweb.py @@ -5,7 +5,7 @@ from cgi import escape from logger import * from flup.server.fcgi import WSGIServer -elist = query.ExitList(csv_file = "bel_export.csv") +elist = query.ExitList(filename = "bel_export.csv") port = 8000 log = get_logger("WSGITest")

1 0

[torbel/master] Update the checkweb tool
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit a6d3263bd6c0b9031ba21fca79795d3f04be3521 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 07:57:29 2011 +0200 Update the checkweb tool --- checkweb.py | 38 +++++++++++++++++++++++++++++++------- 1 files changed, 31 insertions(+), 7 deletions(-) diff --git a/checkweb.py b/checkweb.py index 1d7d799..85a99a5 100755 --- a/checkweb.py +++ b/checkweb.py @@ -1,20 +1,46 @@ #!/usr/bin/python import sys, os -import query + +sys.path.append('/home/torbel') +sys.path.append('/home/torbel/torflow') +sys.path.append('/home/torbel/torbel') + +import torbel.query as query from cgi import escape from logger import * from flup.server.fcgi import WSGIServer -elist = query.ExitList(filename = "bel_export.csv") -port = 8000 +import time +from threading import Timer + +import math + +elist = None + +port = 80 log = get_logger("WSGITest") +def update_elist(): + global elist + elist = query.ExitList(filename = "/home/torbel/export/torbel_export.csv", + status_filename = "/home/torbel/export/torbel_export.status") + nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) + if nextUpdate > 0: + log.debug("Scheduling update in %d seconds.", nextUpdate) + Timer(nextUpdate, update_elist, ()).start() + else: + log.notice("Export file is not up-to-date. Trying again in 10 minutes.") + Timer(10*60, update_elist, ()).start() + +update_elist() + def app(environ, start_response): start_response('200 OK', [('Content-Type', 'text/html')]) ipstr = environ["REMOTE_ADDR"] - ip = query.ip_from_string(environ["REMOTE_ADDR"]) - router = elist.is_tor_traffic(ip, port) + ip = query.ip_from_string(ipstr) + dest_ip = query.ip_from_string("78.47.18.106") + router = elist.tor_exit_search(ip, dest_ip, port) yield '<h1>ARE YOU FROM TOR?</h1>' yield '<blink><strong>' @@ -30,5 +56,3 @@ def app(environ, start_response): if __name__ == "__main__": log.debug("Loaded WSGI server (exit list has %d routers).", len(elist.cache_id)) WSGIServer(app).run() - -

1 0

[torbel/master] Actually start the timer for the refresh
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 643548bf399b363e630ba4c62e94c26517a05aec Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 07:58:36 2011 +0200 Actually start the timer for the refresh --- bulklist.fcgi | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index 56dee21..c447903 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -31,10 +31,10 @@ def update_elist(): nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) if nextUpdate > 0: log.debug("Scheduling update in %d seconds.", nextUpdate) - #Timer(nextUpdate, update_elist, ()).start() + Timer(nextUpdate, update_elist, ()).start() else: log.notice("Export file is not up-to-date. Trying again in 10 minutes.") - #Timer(10*60, update_elist, ()).start() + Timer(10*60, update_elist, ()).start() update_elist()

1 0

[torbel/master] Use UTC consistently
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit cbc5d8c69e9386b15a3d366a1b3e9a4b379c96ba Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 08:32:34 2011 +0200 Use UTC consistently --- bulklist.fcgi | 4 +++- checkweb.py | 3 ++- controller.py | 2 +- dnsel/dnsel.py | 5 +++-- doc/data-spec.txt | 6 +++--- query.py | 6 +++--- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi index c447903..92dfcff 100755 --- a/bulklist.fcgi +++ b/bulklist.fcgi @@ -7,6 +7,8 @@ sys.path.append('/home/torbel/torbel') import socket +from calendar import timegm + import torbel.query as query from cgi import escape from urlparse import parse_qs @@ -28,7 +30,7 @@ def update_elist(): global elist elist = query.ExitList(filename = "/home/torbel/export/torbel_export.csv", status_filename = "/home/torbel/export/torbel_export.status") - nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) + nextUpdate = math.ceil(timegm(elist.next_update.timetuple()) - time.time()) if nextUpdate > 0: log.debug("Scheduling update in %d seconds.", nextUpdate) Timer(nextUpdate, update_elist, ()).start() diff --git a/checkweb.py b/checkweb.py index 85a99a5..32c6388 100755 --- a/checkweb.py +++ b/checkweb.py @@ -11,6 +11,7 @@ from logger import * from flup.server.fcgi import WSGIServer import time +from calendar import timegm from threading import Timer import math @@ -25,7 +26,7 @@ def update_elist(): global elist elist = query.ExitList(filename = "/home/torbel/export/torbel_export.csv", status_filename = "/home/torbel/export/torbel_export.status") - nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) + nextUpdate = math.ceil(timegm(elist.next_update.timetuple()) - time.time()) if nextUpdate > 0: log.debug("Scheduling update in %d seconds.", nextUpdate) Timer(nextUpdate, update_elist, ()).start() diff --git a/controller.py b/controller.py index 807ce18..fe6dbf7 100644 --- a/controller.py +++ b/controller.py @@ -524,7 +524,7 @@ class Controller(TorCtl.EventHandler): # Add 30 seconds to the config.export_interval time so clients don't # try to fetch it as we're writing it. - next = datetime.now() + timedelta(0, config.export_interval * 60 + 30) + next = datetime.utcnow() + timedelta(0, config.export_interval * 60 + 30) nextstr = next.strftime("%b %d %Y %H:%M:%S") export_file_prefix = config.export_file_prefix fn = export_file_prefix + ".status" diff --git a/dnsel/dnsel.py b/dnsel/dnsel.py index 2060562..b39230a 100755 --- a/dnsel/dnsel.py +++ b/dnsel/dnsel.py @@ -6,6 +6,7 @@ # and the TorBEL query interface. import os, sys, time from datetime import datetime +from calendar import timegm from zope.interface import implements from twisted.names import server, dns, common, error @@ -32,7 +33,7 @@ class TorDNSServerFactory(server.DNSServerFactory): log.info("Export %s likely stale.", filename) # Set up updates. - nextUpdate = time.mktime(self.el.next_update.timetuple()) - time.time() + nextUpdate = timegm(self.el.next_update.timetuple()) - time.time() if nextUpdate > 0: log.debug("Scheduling first update in %.1f seconds.", nextUpdate) reactor.callLater(nextUpdate, self.update) @@ -45,7 +46,7 @@ class TorDNSServerFactory(server.DNSServerFactory): def update(self): next = self.el.update() - nextUpdate = time.mktime(next.timetuple()) - time.time() + nextUpdate = timegm(next.timetuple()) - time.time() if nextUpdate > 0: log.info("ExitList updated. Next update in %.1f seconds.", nextUpdate) reactor.callLater(nextUpdate, self.update) diff --git a/doc/data-spec.txt b/doc/data-spec.txt index 9c39d87..5d6db11 100644 --- a/doc/data-spec.txt +++ b/doc/data-spec.txt @@ -108,9 +108,9 @@ Status: Draft * NextUpdate "Datetime": The value of this key is a date-time string with strftime(3) format - "%b %d %Y %H:%M:%S". This date represents the absolute earliest time at - which a TorBEL export consumer should attempt to fetch an updated export - file. + "%b %d %Y %H:%M:%S" in UTC. This date represents the absolute earliest + time at which a TorBEL export consumer should attempt to fetch an + updated export file. An example of this key is: NextUpdate "Aug 12 16:51:04" This key must appear exactly once in the status file. diff --git a/query.py b/query.py index 16a6e7a..a5dda85 100644 --- a/query.py +++ b/query.py @@ -212,7 +212,7 @@ class ExitList: def should_update(self): """ Returns True if our updates are out of date. """ - return self.next_update and (datetime.datetime.now() > self.next_update) + return self.next_update and (datetime.datetime.utcnow() > self.next_update) def update(self, force = False): """ Read and update to the latest export and status files. @@ -232,13 +232,13 @@ class ExitList: # Import the status file, if available. if self.status_filename: self.next_update, self.export_files = self.read_status(self.status_filename) - self.stale = (datetime.datetime.now() > self.next_update) + self.stale = (datetime.datetime.utcnow() > self.next_update) else: self.next_update = None self.export_files = None # Record our last update time. - self.last_update = datetime.datetime.now() + self.last_update = datetime.datetime.utcnow() return self.next_update

1 0

[torbel/master] Add metadata to CSV export format.
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 8323b633dbccba0b3e06362cc8961a6f303557d8 Author: Harry Bock <hbock(a)ele.uri.edu> Date: Wed Oct 20 00:43:42 2010 -0400 Add metadata to CSV export format. Add metadata to CSV format specification and implementation (query and export). This allows torbel to know what version of the CSV format to use, allowing for changes in the future and backward compatibility. Not implemented yet for the JSON format. --- __init__.py | 2 ++ controller.py | 4 ++-- doc/data-spec.txt | 13 +++++++++++++ query.py | 24 ++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index e69de29..b5bf7e9 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,2 @@ +# TorBEL export version format. +__export_version__ = 1 diff --git a/controller.py b/controller.py index 29beaff..807ce18 100644 --- a/controller.py +++ b/controller.py @@ -24,7 +24,7 @@ from twisted.internet import reactor from TorCtl import TorCtl, TorUtil # torbel submodules -from torbel import scheduler, network, utils +from torbel import scheduler, network, utils, __export_version__ from torbel.logger import * from torbel.router import RouterRecord @@ -598,7 +598,7 @@ class Controller(TorCtl.EventHandler): csv_file = open(fn_new, "w") out = csv.writer(csv_file, dialect = csv.excel) - + out.writerow(["torbel", __export_version__]) # FIXME: Is it safe to just take the itervalues list? with self.consensus_cache_lock: for router in self.router_cache.itervalues(): diff --git a/doc/data-spec.txt b/doc/data-spec.txt index c5f4014..9c39d87 100644 --- a/doc/data-spec.txt +++ b/doc/data-spec.txt @@ -12,6 +12,8 @@ Status: Draft This document is a work-in-progress and the data format may change quickly during the summer in response to demand and implementation problems. + This document describes TorBEL export data format version 1. + 1. Exported Data 1.1. Records @@ -152,6 +154,17 @@ Status: Draft Fields that contain an escaped double quote are also enclosed in a set of double quotes. + Metadata indicating the CSV export format version must be present on the first + line in the format: + + torbel,VERSION + + Where VERSION is the export format version as an integer, and torbel is + the literal string 'torbel'. Additional metadata fields may be added + in future export versions and consumers MUST handle extraneous fields + on this line not specified in this document. + + Following the metadata line, all remaining lines are router data rows. The fields of each row are, in order: ExitAddress, RouterID, Nickname, LastTestedTimestamp, InConsensus, diff --git a/query.py b/query.py index 69e9ec9..16a6e7a 100644 --- a/query.py +++ b/query.py @@ -10,6 +10,7 @@ import ipaddr import sys from socket import inet_aton, inet_ntoa from logger import * +from torbel import __export_version__ if sys.version_info >= (2,6): import json @@ -189,10 +190,14 @@ class ExitPolicyRule: return "reject " + ip + ":" + port class ExitList: + class ImportError(ValueError): + pass + def __init__(self, filename, status_filename = None): self.cache_ip = {} self.cache_id = {} + self.version = None self.next_update = None self.last_update = None self.export_files = [] @@ -286,6 +291,25 @@ class ExitList: the TorBEL data-spec document. """ reader = csv.reader(infile, dialect = "excel") record = 1 + # Grab metadata row and export format version. + metadata = reader.next() + try: + self.version = int(metadata[1]) + if metadata[0] != "torbel": + raise self.ImportError("Invalid TorBEL export format.") + if self.version > __export_version__: + raise self.ImportError("Export version %d not supported!" % self.version) + + # ValueError will be raised if the first value on the metadata line + # is not an integer. + # IndexError is raised if the metadata line is empty. Not quite sure + # if this is actually possible! + # StopIteration is raised if we try to read from an empty file. + # All of these indicate the TorBEL export file is not actually + # a valid export. + except (ValueError, IndexError, StopIteration): + raise self.ImportError("Invalid TorBEL export format.") + for r in reader: try: data = {

1 0

[torbel/master] Add some new requirements to the README
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit 0b8aaf185c5ee990341df21467007a207973eaaf Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Thu Jul 21 17:45:45 2011 +0200 Add some new requirements to the README --- README | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README b/README index 5e2fa87..eb618ab 100644 --- a/README +++ b/README @@ -20,7 +20,7 @@ Software Requirements The following are required to run TorBEL on your system: - - Tor 0.2.2.13-alpha or higher recommended, with an open + - Tor 0.2.2.30-rc or higher recommended, with an open control port. - Python (>= 2.5 is required, >= 2.6 is recommended and required for JSON support) @@ -52,6 +52,7 @@ is available in your torrc: - FetchDirInfoExtraEarly 1 (Tor >= 0.2.2.13-alpha) - LearnCircuitBuiltTimeout 0 (Tor >= 0.2.2.14-alpha) - WarnUnsafeSocks 0 (Tor >= 0.2.2.14-alpha) + - FetchV2Networkstatus 1 (Tor >= 0.2.2.26-beta) These options significantly affect Tor's operation and thus you should not run other applications that depend on Tor's normal operation with the same instance used by TorBEL. @@ -151,4 +152,4 @@ You can configure TorBEL to log to any combination of stdout, syslog, and a (non-rotated) log file. Logging to file is extremely useful for debugging. Be warned that if your log_level is set to DEBUG or lower, your daemon.log (or wherever your syslog ends up) will get utterly -spammed. You have been warned! \ No newline at end of file +spammed. You have been warned!

1 0

[torbel/master] Add the bulk exit list tool
by sebastian＠torproject.org 04 Sep '11

04 Sep '11

commit affe14eee229adb36b4bef5c5de5afe7e6da6b1b Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sun Sep 4 07:58:04 2011 +0200 Add the bulk exit list tool --- bulklist.fcgi | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 78 insertions(+), 0 deletions(-) diff --git a/bulklist.fcgi b/bulklist.fcgi new file mode 100755 index 0000000..56dee21 --- /dev/null +++ b/bulklist.fcgi @@ -0,0 +1,78 @@ +#!/usr/bin/python +import sys, os + +sys.path.append('/home/torbel') +sys.path.append('/home/torbel/torflow') +sys.path.append('/home/torbel/torbel') + +import socket + +import torbel.query as query +from cgi import escape +from urlparse import parse_qs +from logger import * +from flup.server.fcgi import WSGIServer + +import time +from threading import Timer + +import math + +from ipaddr import IPAddress + +elist = None + +log = get_logger("BulkExitList") + +def update_elist(): + global elist + elist = query.ExitList(filename = "/home/torbel/export/torbel_export.csv", + status_filename = "/home/torbel/export/torbel_export.status") + nextUpdate = math.ceil(time.mktime(elist.next_update.timetuple()) - time.time()) + if nextUpdate > 0: + log.debug("Scheduling update in %d seconds.", nextUpdate) + #Timer(nextUpdate, update_elist, ()).start() + else: + log.notice("Export file is not up-to-date. Trying again in 10 minutes.") + #Timer(10*60, update_elist, ()).start() + +update_elist() + +def app(environ, start_response): + q = parse_qs(environ['QUERY_STRING']) + + + ip = escape(q.get('ip', [''])[0]) + port = escape(q.get('port', [''])[0]) + + if ip != "": + try: + ip = query.ip_from_string(ip) + except socket.error: + ip = 0 + + if ip != 0: + try: + port = int(port) + if port > 65535 or port < 1: + port = 80 + except ValueError: + port = 80 + exits = elist.will_exit_to(ip, port) + + start_response('200 OK', [('Content-Type', 'text/plain')]) + + yield "# This is a list of all Tor exit nodes that can contact %s " % IPAddress(ip) + yield "on Port %d #\n" % port + yield "# This file was generated on %s UTC #\n" % time.asctime(time.gmtime()) + exits.sort() + for i in exits: + yield '%s\n' % IPAddress(i) + else: + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield 'bai' + + +if __name__ == "__main__": + WSGIServer(app).run() +

1 0

← Newer
1
...
73
74
75
76
77
78
79
...
87
Older →

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits September 2011