January 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix bug in basic test deck
by art＠torproject.org 03 Jan '14

03 Jan '14

commit ac28b4c81b2dc0d05c51abcdef33e0124b24ffd4 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 12:19:05 2013 +0100 Fix bug in basic test deck --- data/decks/basic.deck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/decks/basic.deck b/data/decks/basic.deck index b2ede99..1643f5a 100644 --- a/data/decks/basic.deck +++ b/data/decks/basic.deck @@ -7,7 +7,7 @@ pcapfile: null reportfile: null resume: 0 - subargs: [-f, 'httpo://amcq5ldf3vwg22ze.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] + subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] test_file: blocking/http_requests testdeck: null - options:

1 0

[ooni-probe/master] Merge branch 'master' into fix/misc
by art＠torproject.org 03 Jan '14

03 Jan '14

commit 76c4401bc715f0f36d4324faacb5c66c5b331e31 Merge: d3c0b03 7819f1a Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 12:20:40 2013 +0100 Merge branch 'master' into fix/misc * master: Removed dated reference to subversion README.md | 6 ------ 1 file changed, 6 deletions(-)

1 0

[ooni-probe/master] Don't use debug verbosity by default
by art＠torproject.org 03 Jan '14

03 Jan '14

commit d3c0b034f4b4d1e60809f2309a3a6dd907d4970d Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 12:19:18 2013 +0100 Don't use debug verbosity by default --- data/ooniprobe.conf.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index 3128d3d..83c1ebc 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -22,7 +22,7 @@ reports: collector: 'httpo://nkvphnp3p6agi5qq.onion' advanced: geoip_data_dir: /usr/share/ooni/ - debug: true + debug: false # enable if auto detection fails #tor_binary: /usr/sbin/tor #obfsproxy_binary: /usr/bin/obfsproxy

1 0

[ooni-probe/master] Improve output of ooniprobe client
by art＠torproject.org 03 Jan '14

03 Jan '14

commit 9e0df8c96f5c9f309ef2c8d9c9d49cac5f7be14d Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 14:01:12 2013 +0100 Improve output of ooniprobe client --- ooni/director.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 4c53ca6..ed4517c 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -279,7 +279,7 @@ class Director(object): config.tor.socks_port = int(socks_port.values()[0]) config.tor.control_port = int(control_port.values()[0]) - log.debug("Obtained our IP address from a Tor Relay %s" % config.probe_ip) + log.msg("Obtained our IP address from a Tor Relay %s" % config.probe_ip) def setup_failed(failure): log.exception(failure) @@ -296,7 +296,7 @@ class Director(object): return state.post_bootstrap def updates(prog, tag, summary): - log.debug("%d%%: %s" % (prog, summary)) + log.msg("%d%%: %s" % (prog, summary)) tor_config = TorConfig() if config.tor.control_port:

1 0

[ooni-probe/master] If we don't find the content length header set it to None
by art＠torproject.org 03 Jan '14

03 Jan '14

commit 215cd51cc23d018b70683ad540727f70856e6867 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 15:01:04 2013 +0100 If we don't find the content length header set it to None --- ooni/templates/httpt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 04c0708..f9b9d36 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -234,6 +234,10 @@ class HTTPTest(NetTestCase): content_length = response.headers.getRawHeaders('content-length') except IndexError: content_length = None + + if not content_length: + self._processResponseBody(None, request, response, None) + return defer.succeed(None) finished = defer.Deferred() response.deliverBody(BodyReceiver(finished, content_length)) @@ -322,7 +326,6 @@ class HTTPTest(NetTestCase): d = agent.request(request['method'], request['url'], headers, body_producer) - d.addErrback(errback, request) d.addCallback(self._cbResponse, request, headers_processor, body_processor)

1 0

[ooni-probe/master] Properly handle probe IP address lookup failures
by art＠torproject.org 03 Jan '14

03 Jan '14

commit dc48bc3ce9c43de591d60e95617e94c059bfeef0 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 14:06:48 2013 +0100 Properly handle probe IP address lookup failures --- ooni/oonicli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 9405044..061eab3 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -176,7 +176,7 @@ def runWithDirector(): r = failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress, errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper, - errors.CouldNotFindTestCollector) + errors.CouldNotFindTestCollector, errors.ProbeIPUnknown) if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") @@ -201,6 +201,9 @@ def runWithDirector(): log.err("Could not find a valid collector.") log.msg("Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.") + elif isinstance(failure.value, errors.ProbeIPUnknown): + log.err("Failed to lookup probe IP address.") + log.msg("Check your internet connection.") if config.advanced.debug: log.exception(failure)

1 0

[ooni-probe/master] Clean up the output of the HTTP Client
by art＠torproject.org 03 Jan '14

03 Jan '14

commit cb0700a37f28283c1ffaab9a1077e8075f3a1f13 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 14:06:30 2013 +0100 Clean up the output of the HTTP Client --- ooni/geoip.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/geoip.py b/ooni/geoip.py index 8ad124c..796401b 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -3,6 +3,8 @@ import os import random from twisted.web import client, http_headers +client._HTTP11ClientFactory.noisy = False + from ooni.utils.net import userAgents, BodyReceiver from twisted.internet import reactor, defer, protocol

1 0

[ooni-probe/master] Better error handling inside of geoip code
by art＠torproject.org 03 Jan '14

03 Jan '14

commit 7023511a76c492741e995bbb1cb901de0780ad8d Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 17:24:48 2013 +0100 Better error handling inside of geoip code --- ooni/geoip.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ooni/geoip.py b/ooni/geoip.py index 796401b..33766f3 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -33,19 +33,26 @@ def IPToLocation(ipaddr): location = {'city': None, 'countrycode': None, 'asn': None} try: city_dat = GeoIP(city_file) - location['city'] = city_dat.record_by_addr(ipaddr)['city'] - + try: + location['city'] = city_dat.record_by_addr(ipaddr)['city'] + except TypeError: + location['city'] = None + country_dat = GeoIP(country_file) location['countrycode'] = country_dat.country_code_by_addr(ipaddr) asn_dat = GeoIP(asn_file) - location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0] + try: + location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0] + except AttributeError: + location['asn'] = 'AS0' except IOError: - log.err("Could not find GeoIP data files. Go into data/ " - "and run make geoip") + log.err("Could not find GeoIP data files. Go into %s " + "and run make geoip or change the geoip_data_dir " + "in the config file" % config.advanced.geoip_data_dir) raise GeoIPDataFilesNotFound - + return location class HTTPGeoIPLookupper(object):

1 0

[ooni-probe/master] Fix unittests
by art＠torproject.org 03 Jan '14

03 Jan '14

commit 54cfb8baf65ea19ddfa68f4b8c9cb0fe0e9d23b4 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 17:44:25 2013 +0100 Fix unittests --- ooni/tests/test_nettest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 718ad89..d965735 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -6,8 +6,8 @@ from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python.usage import UsageError -from ooni.nettest import NetTest, InvalidOption, MissingRequiredOption -from ooni.nettest import NetTestLoader, FailureToLoadNetTest +from ooni.errors import MissingRequiredOption, InvalidOption, FailureToLoadNetTest +from ooni.nettest import NetTest, NetTestLoader from ooni.tasks import BaseTask from ooni.director import Director

1 0

[ooni-probe/master] Improve the output of the traceroute test
by art＠torproject.org 03 Jan '14

03 Jan '14

commit 249ae4895689dca3a69becb2116acd3dbacfa913 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 17:37:30 2013 +0100 Improve the output of the traceroute test --- ooni/nettests/manipulation/traceroute.py | 12 ++++++------ ooni/templates/scapyt.py | 4 ++-- ooni/utils/txscapy.py | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index 5d27fb7..aaa977a 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -54,7 +54,7 @@ class TracerouteTest(scapyt.BaseScapyTest): with TTLs from 1 until max_ttl. """ def finished(packets, port): - log.debug("Finished running TCP traceroute test on port %s" % port) + log.msg("Finished running TCP traceroute test on port %s" % port) answered, unanswered = packets self.report['test_tcp_traceroute']['hops_'+str(port)] = [] for snd, rcv in answered: @@ -63,7 +63,7 @@ class TracerouteTest(scapyt.BaseScapyTest): 'rtt': rcv.time - snd.time, 'sport': snd[TCP].sport } - log.debug("%s: %s" % (port, report)) + log.msg("%s: %s" % (port, report)) self.report['test_tcp_traceroute']['hops_'+str(port)].append(report) dl = [] @@ -84,7 +84,7 @@ class TracerouteTest(scapyt.BaseScapyTest): payloads with TTLs from 1 until max_ttl. """ def finished(packets, port): - log.debug("Finished running UDP traceroute test on port %s" % port) + log.msg("Finished running UDP traceroute test on port %s" % port) answered, unanswered = packets self.report['test_udp_traceroute']['hops_'+str(port)] = [] for snd, rcv in answered: @@ -93,7 +93,7 @@ class TracerouteTest(scapyt.BaseScapyTest): 'rtt': rcv.time - snd.time, 'sport': snd[UDP].sport } - log.debug("%s: %s" % (port, report)) + log.msg("%s: %s" % (port, report)) self.report['test_udp_traceroute']['hops_'+str(port)].append(report) dl = [] max_ttl, timeout = self.max_ttl_and_timeout() @@ -113,7 +113,7 @@ class TracerouteTest(scapyt.BaseScapyTest): packets with TTLs from 1 until max_ttl. """ def finished(packets): - log.debug("Finished running ICMP traceroute test") + log.msg("Finished running ICMP traceroute test") answered, unanswered = packets self.report['test_icmp_traceroute']['hops'] = [] for snd, rcv in answered: @@ -121,7 +121,7 @@ class TracerouteTest(scapyt.BaseScapyTest): 'address': rcv.src, 'rtt': rcv.time - snd.time } - log.debug("%s" % (report)) + log.msg("%s" % (report)) self.report['test_icmp_traceroute']['hops'].append(report) dl = [] max_ttl, timeout = self.max_ttl_and_timeout() diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index fdc5a24..a4d66c2 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -83,8 +83,8 @@ class BaseScapyTest(NetTestCase): received_packet = rcv if not config.privacy.includeip: - log.msg("Detected you would not like to include your ip in the report") - log.msg("Stripping source and destination IPs from the reports") + log.debug("Detected you would not like to include your ip in the report") + log.debug("Stripping source and destination IPs from the reports") sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1' diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index a5bb841..1012135 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -127,7 +127,11 @@ class ScapyFactory(abstract.FileDescriptor): abstract.FileDescriptor.__init__(self, reactor) if interface == 'auto': - interface = getDefaultIface() + try: + interface = getDefaultIface() + except IfaceError: + log.err("Could not detect the default interface.") + log.msg("Try editing your config file to set it manually.") if not super_socket: super_socket = conf.L3socket(iface=interface, promisc=True, filter='')

1 0