March 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge branch 'master' into feature/bridget
by art＠torproject.org 24 Mar '14

24 Mar '14

commit e718ff2d5d185781d86f089a84f5486b3970f3f0 Merge: e7ac980 064f3e5 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Mar 10 18:51:51 2014 +0100 Merge branch 'master' into feature/bridget * master: Return the correct deferred in director start method. Initialize all report entries in http requests test Add unittests for oonicli. Move the parasitic traceroute function to the experimental tests. Make the getAddresses function more robust Update changelog for 1.0.0 Make the report format match the ooni-spec Clean up the traceroute test Fix bug that leads to not showing the help if an option is not present. Add a Parasitic Traceroute NetTest Refactor Multi Protocol Traceroute, add ParasiticTraceroute Add a ScapyTraceroute ScapyProtocol Get a Tor Controller connection for a tor ooniprobe did not launch. improve hasRawSocketPermission. Conflicts: ooni/nettest.py .travis.yml | 1 + ChangeLog.md | 12 + bin/ooniprobe | 6 +- ooni/deck.py | 15 +- ooni/director.py | 30 ++- ooni/errors.py | 4 + ooni/nettest.py | 16 +- ooni/nettests/blocking/http_requests.py | 9 +- ooni/nettests/experimental/parasitictraceroute.py | 157 ++++--------- ooni/nettests/manipulation/traceroute.py | 165 +++++--------- ooni/oonicli.py | 31 +-- ooni/reporter.py | 13 +- ooni/tasks.py | 2 +- ooni/templates/scapyt.py | 2 +- ooni/tests/__init__.py | 1 + ooni/tests/mocks.py | 3 - ooni/tests/test_director.py | 48 ++-- ooni/tests/test_oonicli.py | 115 ++++++++++ ooni/tests/test_templates.py | 4 - ooni/utils/log.py | 37 +++- ooni/utils/txscapy.py | 242 ++++++++++++++++++++- 21 files changed, 594 insertions(+), 319 deletions(-) diff --cc ooni/nettest.py index 520fcf8,ce7831e..6a182ca --- a/ooni/nettest.py +++ b/ooni/nettest.py @@@ -504,12 -496,8 +504,15 @@@ class NetTest(object) self.state = NetTestState(self.done) + def __str__(self): + return ' '.join(tc.name for tc, _ in self.testCases) ++ + def doneNetTest(self, result): + print "Summary for %s" % self.testDetails['test_name'] + print "------------" + "-"*len(self.testDetails['test_name']) + for test_class in self.testClasses: + test_instance = test_class() + test_instance.displaySummary(self.summary) def doneReport(self, report_results): """

1 0

[ooni-probe/master] Only start Tor when it is needed.
by art＠torproject.org 24 Mar '14

24 Mar '14

commit 8bceb1b2632bb1a3a62d1dfa1d57519262918854 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Mar 8 16:34:13 2014 +0100 Only start Tor when it is needed. --- ooni/deck.py | 4 ++ ooni/director.py | 4 +- ooni/nettest.py | 51 +++----------------- ooni/nettests/blocking/dnsconsistency.py | 2 + ooni/nettests/blocking/http_requests.py | 2 + ooni/nettests/blocking/tcp_connect.py | 4 +- ooni/nettests/experimental/dns_injection.py | 2 + ooni/nettests/experimental/domclass_collector.py | 2 + .../experimental/http_filtering_bypassing.py | 3 ++ .../experimental/http_keyword_filtering.py | 2 + ooni/nettests/experimental/http_trix.py | 2 + .../experimental/http_uk_mobile_networks.py | 5 +- ooni/nettests/experimental/keyword_filtering.py | 2 + ooni/nettests/experimental/parasitictraceroute.py | 2 + ooni/nettests/experimental/script.py | 2 + ooni/nettests/experimental/squid.py | 4 ++ ooni/nettests/experimental/tls_handshake.py | 1 + ooni/nettests/manipulation/captiveportal.py | 2 + ooni/nettests/manipulation/daphne.py | 3 +- ooni/nettests/manipulation/dnsspoof.py | 2 + .../manipulation/http_header_field_manipulation.py | 2 + ooni/nettests/manipulation/http_host.py | 2 + .../manipulation/http_invalid_request_line.py | 2 + ooni/nettests/manipulation/traceroute.py | 5 ++ ooni/nettests/scanning/http_url_list.py | 5 +- ooni/nettests/third_party/netalyzr.py | 2 + ooni/oonicli.py | 9 +++- 27 files changed, 73 insertions(+), 55 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 47edadc..cfd5d73 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -86,6 +86,7 @@ class Deck(InputFile): deckFile=None, decks_directory=config.decks_directory): self.id = deck_hash + self.requiresTor = False self.bouncer = '' self.netTestLoaders = [] self.inputs = [] @@ -135,12 +136,15 @@ class Deck(InputFile): return False try: net_test_loader.checkOptions() + if net_test_loader.requiresTor: + self.requiresTor = True except e.MissingRequiredOption, missing_options: if not self.bouncer: raise for missing_option in missing_options.message: if not has_test_helper(missing_option): raise + self.requiresTor = True self.netTestLoaders.append(net_test_loader) @defer.inlineCallbacks diff --git a/ooni/director.py b/ooni/director.py index e4fc261..16c2069 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -118,10 +118,10 @@ class Director(object): return nettests @defer.inlineCallbacks - def start(self): + def start(self, start_tor=False): self.netTests = self.getNetTests() - if config.advanced.start_tor: + if config.advanced.start_tor and start_tor: yield self.startTor() elif config.tor.control_port: log.msg("Connecting to Tor Control Port...") diff --git a/ooni/nettest.py b/ooni/nettest.py index 61b327e..8ac9340 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -38,44 +38,6 @@ def get_test_methods(item, method_prefix="test_"): pass return test_cases -def loadNetTestString(net_test_string): - """ - Load NetTest from a string. - WARNING input to this function *MUST* be sanitized and *NEVER* be - untrusted. - Failure to do so will result in code exec. - - net_test_string: - - a string that contains the net test to be run. - """ - net_test_file_object = StringIO(net_test_string) - - ns = {} - test_cases = [] - exec net_test_file_object.read() in ns - for item in ns.itervalues(): - test_cases.extend(get_test_methods(item)) - - if not test_cases: - raise e.NoTestCasesFound - - return test_cases - -def loadNetTestFile(net_test_file): - """ - Load NetTest from a file. - """ - test_cases = [] - module = filenameToModule(net_test_file) - for __, item in getmembers(module): - test_cases.extend(get_test_methods(item)) - - if not test_cases: - raise e.NoTestCasesFound - - return test_cases - def getTestClassFromFile(net_test_file): """ Will return the first class that is an instance of NetTestCase. @@ -173,20 +135,18 @@ def getNetTestInformation(net_test_file): class NetTestLoader(object): method_prefix = 'test' collector = None + requiresTor = False def __init__(self, options, test_file=None, test_string=None): self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$") self.options = options - self.testCases, test_cases = None, None + self.testCases = [] if test_file: - test_cases = loadNetTestFile(test_file) + self.loadNetTestFile(test_file) elif test_string: - test_cases = loadNetTestString(test_string) + self.loadNetTestString(test_string) - if test_cases: - self.setupTestCases(test_cases) - @property def requiredTestHelpers(self): required_test_helpers = [] @@ -410,6 +370,8 @@ class NetTestLoader(object): test_instance = klass() if test_instance.requiresRoot: checkForRoot() + if test_instance.requiresTor: + self.requiresTor = True test_instance._checkRequiredOptions() test_instance._checkValidOptions() @@ -671,6 +633,7 @@ class NetTestCase(object): requiredTestHelpers = {} requiredOptions = [] requiresRoot = False + requiresTor = False localOptions = {} def _setUp(self): diff --git a/ooni/nettests/blocking/dnsconsistency.py b/ooni/nettests/blocking/dnsconsistency.py index 49e5360..5d17741 100644 --- a/ooni/nettests/blocking/dnsconsistency.py +++ b/ooni/nettests/blocking/dnsconsistency.py @@ -46,6 +46,8 @@ class DNSConsistencyTest(dnst.DNSTest): 'Input file of list of hostnames to attempt to resolve'] requiredTestHelpers = {'backend': 'dns'} + requiresRoot = False + requiresTor = False usageOptions = UsageOptions requiredOptions = ['backend', 'file'] diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index 17859f5..84c80dc 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -36,6 +36,8 @@ class HTTPRequestsTest(httpt.HTTPTest): inputFile = ['file', 'f', None, 'List of URLS to perform GET and POST requests to'] + requiresRoot = False + requiresTor = False # These values are used for determining censorship based on response body # lengths diff --git a/ooni/nettests/blocking/tcp_connect.py b/ooni/nettests/blocking/tcp_connect.py index 129a2d5..84ceb2d 100644 --- a/ooni/nettests/blocking/tcp_connect.py +++ b/ooni/nettests/blocking/tcp_connect.py @@ -20,7 +20,9 @@ class TCPConnectTest(nettest.NetTestCase): version = "0.1" inputFile = ['file', 'f', None, 'File containing the IP:PORT combinations to be tested, one per line'] - + + requiresTor = False + requiresRoot = False requiredOptions = ['file'] def test_connect(self): """ diff --git a/ooni/nettests/experimental/dns_injection.py b/ooni/nettests/experimental/dns_injection.py index 97233cf..cf25539 100644 --- a/ooni/nettests/experimental/dns_injection.py +++ b/ooni/nettests/experimental/dns_injection.py @@ -30,6 +30,8 @@ class DNSInjectionTest(dnst.DNSTest): usageOptions = UsageOptions requiredOptions = ['resolver', 'file'] + requiresRoot = False + requiresTor = False def setUp(self): self.resolver = (self.localOptions['resolver'], 53) diff --git a/ooni/nettests/experimental/domclass_collector.py b/ooni/nettests/experimental/domclass_collector.py index c1866f2..6384e62 100644 --- a/ooni/nettests/experimental/domclass_collector.py +++ b/ooni/nettests/experimental/domclass_collector.py @@ -20,6 +20,8 @@ class DOMClassCollector(httpt.HTTPTest): followRedirects = True inputFile = ['file', 'f', None, 'The list of urls to build a domclass for'] + requiresTor = False + requiresRoot = False def test_collect(self): if self.input: diff --git a/ooni/nettests/experimental/http_filtering_bypassing.py b/ooni/nettests/experimental/http_filtering_bypassing.py index dc103db..1faed37 100644 --- a/ooni/nettests/experimental/http_filtering_bypassing.py +++ b/ooni/nettests/experimental/http_filtering_bypassing.py @@ -20,10 +20,13 @@ class HTTPFilteringBypass(tcpt.TCPTest): usageOptions = UsageOptions requiredOptions = ['backend'] + requiresRoot = False + requiresTor = False def setUp(self): self.port = int(self.localOptions['backendport']) self.address = self.localOptions['backend'] + self.report['tampering'] = None def check_for_manipulation(self, response, payload): log.debug("Checking if %s == %s" % (response, payload)) diff --git a/ooni/nettests/experimental/http_keyword_filtering.py b/ooni/nettests/experimental/http_keyword_filtering.py index 0ae9c52..cbf12d1 100644 --- a/ooni/nettests/experimental/http_keyword_filtering.py +++ b/ooni/nettests/experimental/http_keyword_filtering.py @@ -26,6 +26,8 @@ class HTTPKeywordFiltering(httpt.HTTPTest): inputFile = ['file', 'f', None, 'List of keywords to use for censorship testing'] usageOptions = UsageOptions + requiresTor = False + requiresRoot = False requiredOptions = ['backend'] diff --git a/ooni/nettests/experimental/http_trix.py b/ooni/nettests/experimental/http_trix.py index 85a4ba2..b5c9ae2 100644 --- a/ooni/nettests/experimental/http_trix.py +++ b/ooni/nettests/experimental/http_trix.py @@ -16,6 +16,8 @@ class HTTPTrix(tcpt.TCPTest): authors = "Arturo Filastò" usageOptions = UsageOptions + requiresTor = False + requiresRoot = False requiredOptions = ['backend'] def setUp(self): diff --git a/ooni/nettests/experimental/http_uk_mobile_networks.py b/ooni/nettests/experimental/http_uk_mobile_networks.py index 784a9e9..cc00fdc 100644 --- a/ooni/nettests/experimental/http_uk_mobile_networks.py +++ b/ooni/nettests/experimental/http_uk_mobile_networks.py @@ -33,6 +33,8 @@ class HTTPUKMobileNetworksTest(httpt.HTTPTest): inputFile = ['urls', 'f', None, 'List of urls one per line to test for censorship'] requiredOptions = ['urls'] + requiresRoot = False + requiresTor = False def testPattern(self, value, pattern, type): if type == 'eq': @@ -80,6 +82,3 @@ class HTTPUKMobileNetworksTest(httpt.HTTPTest): log.msg("Testing rules %s" % rules) redirect = self.testRules(rules, location) self.report['redirect'] = redirect - - - diff --git a/ooni/nettests/experimental/keyword_filtering.py b/ooni/nettests/experimental/keyword_filtering.py index 9eec4ff..0937efd 100644 --- a/ooni/nettests/experimental/keyword_filtering.py +++ b/ooni/nettests/experimental/keyword_filtering.py @@ -26,6 +26,8 @@ class KeywordFiltering(scapyt.BaseScapyTest): inputFile = ['file', 'f', None, 'List of keywords to use for censorship testing'] + requiresRoot = True + requiresTor = False def test_tcp_keyword_filtering(self): """ diff --git a/ooni/nettests/experimental/parasitictraceroute.py b/ooni/nettests/experimental/parasitictraceroute.py index c8aa3ed..9526455 100644 --- a/ooni/nettests/experimental/parasitictraceroute.py +++ b/ooni/nettests/experimental/parasitictraceroute.py @@ -13,6 +13,8 @@ class ParasiticTracerouteTest(scapyt.BaseScapyTest): version = '0.1' samplePeriod = 40 + requiresTor = False + requiresRoot = False def setUp(self): self.report['parasitic_traceroute'] = {} diff --git a/ooni/nettests/experimental/script.py b/ooni/nettests/experimental/script.py index 4772f65..c03077a 100644 --- a/ooni/nettests/experimental/script.py +++ b/ooni/nettests/experimental/script.py @@ -66,6 +66,8 @@ class Script(nettest.NetTestCase): usageOptions = UsageOptions requiredOptions = ['interpreter', 'script'] + requiresRoot = False + requiresTor = False def test_run_script(self): """ diff --git a/ooni/nettests/experimental/squid.py b/ooni/nettests/experimental/squid.py index 777bc3e..17255e5 100644 --- a/ooni/nettests/experimental/squid.py +++ b/ooni/nettests/experimental/squid.py @@ -23,6 +23,10 @@ class SquidTest(httpt.HTTPTest): #inputFile = ['urls', 'f', None, 'Urls file'] inputs =['http://google.com'] + + requiresRoot = False + requiresTor = False + def test_cacheobject(self): """ This detects the presence of a squid transparent HTTP proxy by sending diff --git a/ooni/nettests/experimental/tls_handshake.py b/ooni/nettests/experimental/tls_handshake.py index f775935..a38983a 100644 --- a/ooni/nettests/experimental/tls_handshake.py +++ b/ooni/nettests/experimental/tls_handshake.py @@ -143,6 +143,7 @@ class HandshakeTest(nettest.NetTestCase): version = '0.0.3' requiresRoot = False + requiresTor = False usageOptions = HandshakeOptions host = None diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index afbdfeb..d6d8040 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -72,6 +72,8 @@ class CaptivePortal(nettest.NetTestCase): version = '0.2' author = "Isis Lovecruft" usageOptions = UsageOptions + requiresRoot = False + requiresTor = False def http_fetch(self, url, headers={}): """ diff --git a/ooni/nettests/manipulation/daphne.py b/ooni/nettests/manipulation/daphne.py index 936111a..545514b 100644 --- a/ooni/nettests/manipulation/daphne.py +++ b/ooni/nettests/manipulation/daphne.py @@ -69,7 +69,8 @@ class daphn3Test(nettest.NetTestCase): 'Specify the pcap or YAML file to be used as input to the test'] #requiredOptions = ['file'] - + requiresRoot = False + requiresTor = False steps = None def inputProcessor(self, filename): diff --git a/ooni/nettests/manipulation/dnsspoof.py b/ooni/nettests/manipulation/dnsspoof.py index 82fb325..15aad07 100644 --- a/ooni/nettests/manipulation/dnsspoof.py +++ b/ooni/nettests/manipulation/dnsspoof.py @@ -32,6 +32,8 @@ class DNSSpoof(scapyt.ScapyTest): requiredTestHelpers = {'backend': 'dns'} requiredOptions = ['hostname', 'resolver'] + requiresRoot = True + requiresTor = False def setUp(self): self.resolverAddr, self.resolverPort = self.localOptions['resolver'].split(':') diff --git a/ooni/nettests/manipulation/http_header_field_manipulation.py b/ooni/nettests/manipulation/http_header_field_manipulation.py index 93beb25..3f1b4ed 100644 --- a/ooni/nettests/manipulation/http_header_field_manipulation.py +++ b/ooni/nettests/manipulation/http_header_field_manipulation.py @@ -51,6 +51,8 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): requiredTestHelpers = {'backend': 'http-return-json-headers'} requiredOptions = ['backend'] + requiresTor = False + requiresRoot = False def get_headers(self): headers = {} diff --git a/ooni/nettests/manipulation/http_host.py b/ooni/nettests/manipulation/http_host.py index 5e77415..7207c10 100644 --- a/ooni/nettests/manipulation/http_host.py +++ b/ooni/nettests/manipulation/http_host.py @@ -47,6 +47,8 @@ class HTTPHost(httpt.HTTPTest): requiredTestHelpers = {'backend': 'http-return-json-headers'} requiredOptions = ['backend'] + requiresTor = False + requiresRoot = False def setUp(self): self.report['transparent_http_proxy'] = False diff --git a/ooni/nettests/manipulation/http_invalid_request_line.py b/ooni/nettests/manipulation/http_invalid_request_line.py index 2812e45..6769269 100644 --- a/ooni/nettests/manipulation/http_invalid_request_line.py +++ b/ooni/nettests/manipulation/http_invalid_request_line.py @@ -28,6 +28,8 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): requiredTestHelpers = {'backend': 'tcp-echo'} requiredOptions = ['backend'] + requiresRoot = False + requiresTor = False def setUp(self): self.port = int(self.localOptions['backendport']) diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index e74c864..66e0a34 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -24,8 +24,13 @@ class UsageOptions(usage.Options): class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" + description = "Performs a UDP, TCP, ICMP traceroute with destination port number set to 0, 22, 23, 53, 80, 123, 443, 8080 and 65535" + requiredTestHelpers = {'backend': 'traceroute'} + requiresRoot = True + requiresTor = False + usageOptions = UsageOptions dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535] version = "0.3" diff --git a/ooni/nettests/scanning/http_url_list.py b/ooni/nettests/scanning/http_url_list.py index 0accaae..98e4ba0 100644 --- a/ooni/nettests/scanning/http_url_list.py +++ b/ooni/nettests/scanning/http_url_list.py @@ -29,6 +29,9 @@ class HTTPURLList(httpt.HTTPTest): usageOptions = UsageOptions + requiresRoot = False + requiresTor = False + inputFile = ['file', 'f', None, 'List of URLS to perform GET and POST requests to'] @@ -94,5 +97,3 @@ class HTTPURLList(httpt.HTTPTest): def test_put(self): return self.doRequest(self.url, method="PUT") - - diff --git a/ooni/nettests/third_party/netalyzr.py b/ooni/nettests/third_party/netalyzr.py index 9b21831..63ab3c7 100644 --- a/ooni/nettests/third_party/netalyzr.py +++ b/ooni/nettests/third_party/netalyzr.py @@ -14,6 +14,8 @@ from twisted.internet import reactor, threads, defer class NetalyzrWrapperTest(nettest.NetTestCase): name = "NetalyzrWrapper" + requiresRoot = False + requiresTor = False def setUp(self): cwd = os.path.abspath(os.path.join(os.path.abspath(__file__), '..')) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 935d091..ed7b297 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -133,8 +133,6 @@ def runWithDirector(logging=True, start_tor=True): sys.exit(0) - d = director.start() - #XXX: This should mean no bouncer either! if global_options['no-collector']: log.msg("Not reporting using a collector") @@ -143,6 +141,11 @@ def runWithDirector(logging=True, start_tor=True): deck = Deck() deck.bouncer = global_options['bouncer'] + start_tor = deck.requiresTor + if global_options['bouncer']: + start_tor = True + if global_options['collector']: + start_tor = True try: if global_options['testdeck']: @@ -165,6 +168,8 @@ def runWithDirector(logging=True, start_tor=True): print net_test_loader.usageOptions().getUsage() sys.exit(2) + d = director.start(start_tor=start_tor) + def setup_nettest(_): try: return deck.setup()

1 0

[ooni-probe/master] The postProcessor must return the report
by art＠torproject.org 24 Mar '14

24 Mar '14

commit f4f2e8ec267550d992c3c3ab7fbbc530601dd32b Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 11 00:38:46 2014 +0100 The postProcessor must return the report --- ooni/nettests/blocking/bridge_reachability.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index 0d985ae..393c02d 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -62,6 +62,7 @@ class BridgeReachability(nettest.NetTestCase): self.summary['successes'].append(details) else: self.summary['failures'].append(details) + return self.report def displaySummary(self, summary): successful_count = {}

1 0

[ooni-probe/master] Include the transport type in the summary.
by art＠torproject.org 24 Mar '14

24 Mar '14

commit cb48781f10986cdc6a3af1121732923f2470991b Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Mar 10 23:13:45 2014 +0100 Include the transport type in the summary. --- ooni/nettests/blocking/bridge_reachability.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index d55ba29..0d985ae 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -74,8 +74,8 @@ class BridgeReachability(nettest.NetTestCase): count(summary['successes'], successful_count) count(summary['failures'], failure_count) - working_bridges = ', '.join([x['address'] for x in summary['successes']]) - failing_bridges = ', '.join([x['address'] + " (at %s%%)" % x['tor_progress'] for x in summary['failures']]) + working_bridges = ', '.join(["%s %s" % (x['transport_name'], x['address']) for x in summary['successes']]) + failing_bridges = ', '.join(["%s %s (at %s%%)" % (x['transport_name'], x['address'], x['tor_progress']) for x in summary['failures']]) print "Total successes: %d" % len(summary['successes']) print "Total failures: %d" % len(summary['failures'])

1 0

[ooni-probe/master] Terminate the tor process cleanly after we have run a test.
by art＠torproject.org 24 Mar '14

24 Mar '14

commit d34931290c0ed0887ad77f83a4cb77926a47e998 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 11 18:11:02 2014 +0100 Terminate the tor process cleanly after we have run a test. --- ooni/nettests/blocking/bridge_reachability.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index 393c02d..892ed02 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -5,7 +5,7 @@ import subprocess from distutils.spawn import find_executable from twisted.python import usage -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, error import txtorcon @@ -135,6 +135,10 @@ class BridgeReachability(nettest.NetTestCase): progress_updates=updates) @d.addCallback def setup_complete(proto): + try: + proto.transport.signalProcess('TERM') + except error.ProcessExitedAlready: + proto.transport.loseConnection() log.msg("Successfully connected to %s" % self.bridge) self.report['success'] = True

1 0

[ooni-probe/master] Start crond
by art＠torproject.org 24 Mar '14

24 Mar '14

commit f7b67e74fa2f80a0322ede9afa0605a73213ca6c Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 12 18:20:50 2014 +0100 Start crond --- scripts/install_on_planetlab.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install_on_planetlab.sh b/scripts/install_on_planetlab.sh index 3be5872..7fb1af8 100644 --- a/scripts/install_on_planetlab.sh +++ b/scripts/install_on_planetlab.sh @@ -141,3 +141,4 @@ mkdir /home/$USER/bridge_reachability/ # Add cronjob to run ooniprobe daily { crontab -l; echo "0 0 * * * ooniprobe -c httpo://e2nl5qgtkzp7cibx.onion blocking/bridge_reachability -f /home/$USER/bridge_reachability/bridges.txt -t 300"; } | crontab +sudo /etc/init.d/crond start

1 0

[ooni-probe/master] Add required package libpcap0.8-dev
by art＠torproject.org 24 Mar '14

24 Mar '14

commit 3c240ee7a403c7c7b93e4af2c2c17cbb898338c4 Author: anadahz <kojgelo(a)inbox.com> Date: Fri Mar 14 15:54:03 2014 +0100 Add required package libpcap0.8-dev --- setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-dependencies.sh b/setup-dependencies.sh index 9830c34..a6867c7 100755 --- a/setup-dependencies.sh +++ b/setup-dependencies.sh @@ -896,7 +896,7 @@ case $DISTRO_VERSION in echo "[+] Updating OS package list..."; sudo sudo apt-get update 2>&1 > /dev/null; echo "[+] Installing packages for your system..."; - DO "sudo apt-get -y install curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap python-dnspython tor tor-geoipdb libgeoip-dev" "0" + DO "sudo apt-get -y install curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap python-dnspython tor tor-geoipdb libgeoip-dev libpcap0.8-dev" "0" if [ "$PRIV_MODE" -eq "0" ]; then echo "[+] Using virtualenvironment..."

1 0

[ooni-probe/master] Bump to version 1.0.1
by art＠torproject.org 24 Mar '14

24 Mar '14

commit 743cf208ebcb1bda3b8eebc84a8062c0c4afaf40 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 14 12:05:41 2014 +0100 Bump to version 1.0.1 * Update Changelog * Update setup script * Include LICENSE in sdist --- ChangeLog.md | 11 +++++++++++ LICENSE | 2 +- MANIFEST.in | 2 +- nettests/NETTESTS_HAVE_MOVED.txt | 1 - ooni/__init__.py | 4 ++-- setup.py | 12 +++++------- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index f79a219..fda5db1 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,14 @@ +## v1.0.1 + +* Fix bugs in the traceroute test that lead to not all packets being collected. + +* All values inside of http_requests test are now initialized inside of setUp. + +* Fix a bug that lead to the input value of the report not being set in some + circumstances. + +* Add bridge_reachability test + ## v1.0.0 * Add bouncer support for discovering test helpers and collectors diff --git a/LICENSE b/LICENSE index 43eb027..a45daeb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012-2013, Jacob Appelbaum, Aaron Gibson, Arturo Filastò, Isis Lovecruft, The Tor Project +Copyright (c) 2012-2014, Jacob Appelbaum, Aaron Gibson, Arturo Filastò, Isis Lovecruft, The Tor Project All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/MANIFEST.in b/MANIFEST.in index 92586f8..a2e2625 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include README.md ChangeLog.md requirements.txt +include README.md ChangeLog.md requirements.txt LICENSE recursive-include data/decks * include data/inputs/ include data/GeoIP.dat diff --git a/nettests/NETTESTS_HAVE_MOVED.txt b/nettests/NETTESTS_HAVE_MOVED.txt deleted file mode 100644 index 604f2a5..0000000 --- a/nettests/NETTESTS_HAVE_MOVED.txt +++ /dev/null @@ -1 +0,0 @@ -look inside data/nettests/ for all the nettests diff --git a/ooni/__init__.py b/ooni/__init__.py index 82cb67b..86f02b3 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- -__author__ = "Arturo Filastò" -__version__ = "1.0.0" +__author__ = "Open Observatory of Network Interference" +__version__ = "1.0.1" __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', diff --git a/setup.py b/setup.py index 0082365..1410dd7 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python #-*- coding: utf-8 -*- -from ooni import __version__ +from ooni import __version__, __author__ import urllib2 import os import gzip @@ -49,9 +49,9 @@ if hasattr(sys, 'real_prefix'): with open(pj('data', 'ooniprobe.conf.sample.new'), 'w+') as w: with open(pj('data', 'ooniprobe.conf.sample')) as f: for line in f: - if line.startswith(' data_dir: /usr/share/ooni'): + if line.startswith(' data_dir: '): w.write(' data_dir: %s\n' % usr_share_path) - elif line.startswith(' geoip_data_dir: /usr/share/'): + elif line.startswith(' geoip_data_dir: '): w.write(' geoip_data_dir: %s\n' % usr_share_path) else: w.write(line) @@ -88,9 +88,7 @@ for root, dirs, file_names in os.walk('data/'): data_files.append([pj(usr_share_path, root.replace('data/', '')), files]) install_requires = [] -dependency_links = [ - 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe' -] +dependency_links = [] with open('requirements.txt') as f: for line in f: if line.startswith("#"): @@ -103,7 +101,7 @@ with open('requirements.txt') as f: setup( name="ooniprobe", version=__version__, - author="Open Observatory of Network Interference", + author=__author__, author_email = "ooni-dev(a)torproject.org", url="https://ooni.torproject.org/", package_dir={'ooni': 'ooni'},

1 0

[ooni-probe/master] Update cron related functions for installing on planetlab
by art＠torproject.org 24 Mar '14

24 Mar '14

commit 97ae1d0b6acee2e2ca1248f4fcd0020e50752af5 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 14 13:25:16 2014 +0100 Update cron related functions for installing on planetlab --- scripts/install_on_planetlab.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/scripts/install_on_planetlab.sh b/scripts/install_on_planetlab.sh index 7fb1af8..123851f 100644 --- a/scripts/install_on_planetlab.sh +++ b/scripts/install_on_planetlab.sh @@ -76,22 +76,22 @@ prog="tor" start() { - echo -n $"Starting $prog: " - daemon $prog --runasdaemon 1 && success || failure - RETVAL=$? + echo -n $"Starting \$prog: " + daemon \$prog --runasdaemon 1 && success || failure + RETVAL=\$? echo - return $RETVAL + return \$RETVAL } stop() { - echo -n $"Stopping $prog: " - killall $prog - RETVAL=$? + echo -n $"Stopping \$prog: " + killall \$prog + RETVAL=\$? echo - return $RETVAL + return \$RETVAL } -case "$1" in +case "\$1" in start) start ;; @@ -103,10 +103,10 @@ case "$1" in start ;; *) - echo $"Usage: $0 {start|stop|restart}" + echo $"Usage: \$0 {start|stop|restart}" RETVAL=3 esac -exit $RETVAL +exit \$RETVAL EOF sudo mv tor.init /etc/init.d/tor sudo chmod +x /etc/init.d/tor @@ -132,7 +132,7 @@ sudo pip install https://github.com/hellais/pyopenssl/archive/fix/openssl0.9.8co # Install ooniprobe and obfsproxy sudo pip install https://github.com/TheTorProject/ooni-probe/archive/master.zip sudo pip install obfsproxy -ooniprobe +/usr/local/bin/ooniprobe # Update the Tor running in ooniprobe cat /usr/share/ooni/ooniprobe.conf.sample | sed s/'start_tor: true'/'start_tor: false'/ | sed s/'#socks_port: 8801'/'socks_port: 9050'/ > ~/.ooni/ooniprobe.conf @@ -140,5 +140,7 @@ cat /usr/share/ooni/ooniprobe.conf.sample | sed s/'start_tor: true'/'start_tor: mkdir /home/$USER/bridge_reachability/ # Add cronjob to run ooniprobe daily -{ crontab -l; echo "0 0 * * * ooniprobe -c httpo://e2nl5qgtkzp7cibx.onion blocking/bridge_reachability -f /home/$USER/bridge_reachability/bridges.txt -t 300"; } | crontab +{ crontab -l; echo "0 0 * * * /usr/local/bin/ooniprobe -c httpo://e2nl5qgtkzp7cibx.onion blocking/bridge_reachability -f /home/$USER/bridge_reachability/bridges.txt -t 300"; } | crontab sudo /etc/init.d/crond start +sudo /sbin/chkconfig crond on +sudo chmod 777 /var/mail

1 0

[ooni-probe/master] Consider the config file collector option.
by art＠torproject.org 24 Mar '14

24 Mar '14

commit aa766dcb4520fc2c4c1b37444f32d1dd535e4ea0 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 14 15:52:33 2014 +0100 Consider the config file collector option. --- data/ooniprobe.conf.sample | 2 +- ooni/oonicli.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index d1fd847..d32248f 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -19,7 +19,7 @@ privacy: reports: # This is a packet capture file (.pcap) to load as a test: pcap: null - collector: 'httpo://nkvphnp3p6agi5qq.onion' + collector: null advanced: geoip_data_dir: /usr/share/ooni/geoip debug: false diff --git a/ooni/oonicli.py b/ooni/oonicli.py index ed7b297..4fed4c7 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -232,6 +232,8 @@ def runWithDirector(logging=True, start_tor=True): if not global_options['no-collector']: if global_options['collector']: collector = global_options['collector'] + elif config.reports['collector']: + collector = config.report['collector'] elif net_test_loader.collector: collector = net_test_loader.collector

1 0