August 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Bump bridge reachability test version
by art＠torproject.org 19 Aug '14

19 Aug '14

commit fc2e90734cf7d16b4cfb3bbeb0523d682b67febe Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 19 13:49:05 2014 +0200 Bump bridge reachability test version --- ooni/nettests/blocking/bridge_reachability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index f4d5bf1..efe0b7b 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -28,7 +28,7 @@ class BridgeReachability(nettest.NetTestCase): description = "A test for checking if bridges are reachable " \ "from a given location." author = "Arturo Filastò" - version = "0.1" + version = "0.1.1" usageOptions = UsageOptions

1 0

[ooni-probe/master] Implement unittests for report lists.
by art＠torproject.org 19 Aug '14

19 Aug '14

commit fbc97e15711752a9ec7eef9121a4910147bc9fa3 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 14:37:13 2014 +0200 Implement unittests for report lists. Spot a bug and fix it. --- ooni/reporter.py | 4 ++-- ooni/tests/test_reporter.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index f0f8822..e0e92c6 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -422,7 +422,7 @@ class OONIBReportLog(object): for report_file, value in report_log.items(): if value['status'] in ('created'): try: - os.kill(value['pid']) + os.kill(value['pid'], 0) except: reports.append((report_file, value)) return reports @@ -434,7 +434,7 @@ class OONIBReportLog(object): for report_file, value in report_log.items(): if value['status'] in ('created'): try: - os.kill(value['pid']) + os.kill(value['pid'], 0) reports.append((report_file, value)) except: pass diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 06659fd..9aae88c 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -161,3 +161,14 @@ class TestOONIBReportLog(unittest.TestCase): report = yaml.safe_load(f) assert "path_to_my_report.yaml" in report assert report["path_to_my_report.yaml"]["status"] == "creation-failed" + + @defer.inlineCallbacks + def test_list_reports(self): + yield self.report_log.creation_failed("failed_report.yaml", + 'httpo://foo.onion') + yield self.report_log.created("created_report.yaml", + 'httpo://foo.onion', 'XXXX') + + assert len(self.report_log.reports_in_progress) == 1 + assert len(self.report_log.reports_incomplete) == 0 + assert len(self.report_log.reports_to_upload) == 1

1 0

[ooni-probe/master] Make it possible to run ooniprobe also without having dnet installed.
by art＠torproject.org 19 Aug '14

19 Aug '14

commit d2ea103d60d7cae54c75502e45c69038655ebcb7 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 15:35:05 2014 +0200 Make it possible to run ooniprobe also without having dnet installed. --- ooni/director.py | 6 +++++- ooni/oonicli.py | 2 +- ooni/settings.py | 7 ++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 6ef4143..5ffee97 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -105,7 +105,11 @@ class Director(object): for filename in os.listdir(dirname): if is_nettest(filename): net_test_file = os.path.join(dirname, filename) - nettest = getNetTestInformation(net_test_file) + try: + nettest = getNetTestInformation(net_test_file) + except: + log.err("Error processing %s" % filename) + continue nettest['category'] = category.replace('/', '') if nettest['id'] in nettests: diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 4df542f..90dc89a 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -12,7 +12,6 @@ from ooni.settings import config from ooni.director import Director from ooni.deck import Deck, nettest_to_path from ooni.nettest import NetTestLoader -from ooni.utils.txscapy import ScapyFactory from ooni.utils import log, checkForRoot @@ -128,6 +127,7 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True): if config.privacy.includepcap: try: checkForRoot() + from ooni.utils.txscapy import ScapyFactory config.scapyFactory = ScapyFactory(config.advanced.interface) except errors.InsufficientPrivileges: log.err("Insufficient Privileges to capture packets." diff --git a/ooni/settings.py b/ooni/settings.py index 674b4d3..a2a56b8 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -7,7 +7,6 @@ from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol from os.path import abspath, expanduser -from scapy.all import get_if_list from ooni.utils.net import ConnectAndCloseProtocol from ooni import geoip @@ -119,8 +118,10 @@ class OConfig(object): def check_incoherences(self, configuration): incoherent = [] - if configuration['advanced']['interface'] != 'auto' and configuration['advanced']['interface'] not in get_if_list(): - incoherent.append('advanced:interface') + if configuration['advanced']['interface'] != 'auto': + from scapy.all import get_if_list + if configuration['advanced']['interface'] not in get_if_list(): + incoherent.append('advanced:interface') self.log_incoherences(incoherent)

1 0

[ooni-probe/master] Print out the options that are missing.
by art＠torproject.org 19 Aug '14

19 Aug '14

commit 634fcf198efbddc4ac6f261d240c6ae2cfa4e81d Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 15:45:34 2014 +0200 Print out the options that are missing. --- ooni/errors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/errors.py b/ooni/errors.py index 3c9d86f..bcf8c95 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -261,6 +261,8 @@ class MissingRequiredOption(Exception): self.net_test_loader = net_test_loader self.message = message + def __str__(self): + return ','.join(self.message) class OONIUsageError(usage.UsageError): def __init__(self, net_test_loader):

1 0

[ooni-probe/master] Add a unittest for txscapy.
by art＠torproject.org 19 Aug '14

19 Aug '14

commit aac5d653b82db5dca9cfe955749a66a627b9e157 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 14:53:05 2014 +0200 Add a unittest for txscapy. --- ooni/tests/test_txscapy.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/ooni/tests/test_txscapy.py b/ooni/tests/test_txscapy.py index 045eeeb..1dbb8cc 100644 --- a/ooni/tests/test_txscapy.py +++ b/ooni/tests/test_txscapy.py @@ -38,8 +38,10 @@ class TestTxScapy(unittest.TestCase): sender = txscapy.ScapySender() self.scapy_factory.registerProtocol(sender) - packet_sent = IP(dst='8.8.8.8', src='127.0.0.1') / TCP(dport=53, sport=5300) - packet_received = IP(dst='127.0.0.1', src='8.8.8.8') / TCP(sport=53, dport=5300) + packet_sent = IP(dst='8.8.8.8', src='127.0.0.1') / TCP(dport=53, + sport=5300) + packet_received = IP(dst='127.0.0.1', src='8.8.8.8') / TCP(sport=53, + dport=5300) d = sender.startSending([packet_sent]) self.scapy_factory.super_socket.send.assert_called_with(packet_sent) @@ -49,3 +51,17 @@ class TestTxScapy(unittest.TestCase): result = yield d assert result[0][0][0] == packet_sent assert result[0][0][1] == packet_received + + def test_get_addresses(self): + addresses = txscapy.getAddresses() + assert isinstance(addresses, list) + + # @defer.inlineCallbacks + # def test_multi_traceroute(self): + # traceroute = txscapy.MPTraceroute() + # traceroute.timeout = 3 + # self.scapy_factory.registerProtocol(traceroute) + # yield traceroute.ICMPTraceroute('8.8.8.8') + # yield traceroute.TCPTraceroute('8.8.8.8') + # yield traceroute.UDPTraceroute('8.8.8.8') + # self.scapy_factory.super_socket.send.assert_called()

1 0

[ooni-probe/master] Fix oonibclient tests.
by art＠torproject.org 19 Aug '14

19 Aug '14

commit f3314a12f281ae6b2c391351721fd89346247f48 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 15:10:54 2014 +0200 Fix oonibclient tests. --- ooni/tests/test_oonibclient.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 87ea8da..d064d0a 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -2,12 +2,10 @@ import os import shutil import socket -from twisted.trial import unittest from twisted.internet import defer from twisted.web import error from ooni import errors as e -from ooni.utils import log from ooni.settings import config from ooni.oonibclient import OONIBClient from ooni.tests.bases import ConfigTestCase @@ -18,6 +16,7 @@ deck_id = 'd4ae40ecfb3c1b943748cce503ab8233efce7823f3e391058fc0f87829c644ed' class TestOONIBClient(ConfigTestCase): def setUp(self): + super(TestOONIBClient, self).setUp() host = '127.0.0.1' port = 8889 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -33,7 +32,7 @@ class TestOONIBClient(ConfigTestCase): os.mkdir(data_dir) os.mkdir(os.path.join(data_dir, 'inputs')) os.mkdir(os.path.join(data_dir, 'decks')) - except Exception as ex: + except Exception: self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.oonibclient = OONIBClient('http://' + host + ':' + str(port))

1 0

[ooni-probe/master] Handle the case where no command is specified.
by art＠torproject.org 19 Aug '14

19 Aug '14

commit 2e4a26b3632b42a2b43ac3333820f6ec2612977e Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 16:11:48 2014 +0200 Handle the case where no command is specified. --- ooni/report/cli.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ooni/report/cli.py b/ooni/report/cli.py index 6f98574..2523c4f 100644 --- a/ooni/report/cli.py +++ b/ooni/report/cli.py @@ -27,6 +27,11 @@ class Options(usage.Options): sys.exit(0) def parseArgs(self, *args): + if len(args) == 0: + raise usage.UsageError( + "Must specify at least one command" + ) + return self['command'] = args[0] if self['command'] not in ("upload", "status"): raise usage.UsageError( @@ -39,15 +44,6 @@ class Options(usage.Options): self['report_file'] = None -def parse_options(): - options = Options() - try: - options.parseOptions() - except Exception as exc: - print(exc) - return dict(options) - - def tor_check(): if not config.tor.socks_port: print("Currently oonireport requires that you start Tor yourself " @@ -57,7 +53,13 @@ def tor_check(): def run(): config.read_config_file() - options = parse_options() + options = Options() + try: + options.parseOptions() + except Exception as exc: + print("Error: %s" % exc) + print(options) + sys.exit(2) if options['command'] == "upload" and options['report_file']: tor_check() return tool.upload(options['report_file'], @@ -69,3 +71,5 @@ def run(): options['bouncer']) elif options['command'] == "status": return tool.status() + else: + print(options)

1 0

[ooni-probe/master] No collector means no collector.
by art＠torproject.org 19 Aug '14

19 Aug '14

commit 6740b7c87aa8a79d64aaa15d43a68007bf864b8d Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 15:59:10 2014 +0200 No collector means no collector. --- ooni/oonicli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 90dc89a..fdb202d 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -165,11 +165,9 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True): sys.exit(1) global_options["annotations"] = annotations - # XXX: This should mean no bouncer either! if global_options['no-collector']: log.msg("Not reporting using a collector") global_options['collector'] = None - global_options['bouncer'] = None deck = Deck() deck.bouncer = global_options['bouncer']

1 0

[ooni-probe/master] Add extra checks in the body and header comparisons
by art＠torproject.org 19 Aug '14

19 Aug '14

commit 981329235f9b8e8629d993e63e1215dd94e4e0d7 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 16:49:54 2014 +0200 Add extra checks in the body and header comparisons --- ooni/nettests/blocking/http_requests.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index 625f34c..d37489e 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -131,8 +131,10 @@ class HTTPRequestsTest(httpt.HTTPTest): control = measurement.result if experiment and control: - self.compare_body_lengths(len(control.body), - len(experiment.body)) - self.compare_headers(control.headers, - experiment.headers) + if experiment.body and control.body: + self.compare_body_lengths(len(control.body), + len(experiment.body)) + if experiment.headers and control.headers: + self.compare_headers(control.headers, + experiment.headers) return self.report

1 0

[ooni-probe/master] Don't start tor if no collector option is specified and no test helper is needed.
by art＠torproject.org 19 Aug '14

19 Aug '14

commit 5c0bc06f915afbef7a908910ac83a00081063875 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 17:33:53 2014 +0200 Don't start tor if no collector option is specified and no test helper is needed. --- ooni/deck.py | 6 ++++-- ooni/oonicli.py | 4 +--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 3f64064..889e17e 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -90,9 +90,11 @@ def nettest_to_path(path, allow_arbitrary_paths=False): class Deck(InputFile): def __init__(self, deck_hash=None, deckFile=None, - decks_directory=config.decks_directory): + decks_directory=config.decks_directory, + no_collector=False): self.id = deck_hash self.requiresTor = False + self.no_collector = no_collector self.bouncer = '' self.netTestLoaders = [] self.inputs = [] @@ -169,7 +171,7 @@ class Deck(InputFile): required_test_helpers = [] requires_collector = [] for net_test_loader in self.netTestLoaders: - if not net_test_loader.collector: + if not net_test_loader.collector and not self.no_collector: requires_collector.append(net_test_loader) for th in net_test_loader.requiredTestHelpers: diff --git a/ooni/oonicli.py b/ooni/oonicli.py index fdb202d..be242f5 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -169,11 +169,9 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True): log.msg("Not reporting using a collector") global_options['collector'] = None - deck = Deck() + deck = Deck(no_collector=global_options['no-collector']) deck.bouncer = global_options['bouncer'] start_tor = False - if global_options['bouncer']: - start_tor |= True if global_options['collector']: start_tor |= True

1 0