commit 8bceb1b2632bb1a3a62d1dfa1d57519262918854 Author: Arturo Filastò art@fuffa.org Date: Sat Mar 8 16:34:13 2014 +0100
Only start Tor when it is needed. --- ooni/deck.py | 4 ++ ooni/director.py | 4 +- ooni/nettest.py | 51 +++----------------- ooni/nettests/blocking/dnsconsistency.py | 2 + ooni/nettests/blocking/http_requests.py | 2 + ooni/nettests/blocking/tcp_connect.py | 4 +- ooni/nettests/experimental/dns_injection.py | 2 + ooni/nettests/experimental/domclass_collector.py | 2 + .../experimental/http_filtering_bypassing.py | 3 ++ .../experimental/http_keyword_filtering.py | 2 + ooni/nettests/experimental/http_trix.py | 2 + .../experimental/http_uk_mobile_networks.py | 5 +- ooni/nettests/experimental/keyword_filtering.py | 2 + ooni/nettests/experimental/parasitictraceroute.py | 2 + ooni/nettests/experimental/script.py | 2 + ooni/nettests/experimental/squid.py | 4 ++ ooni/nettests/experimental/tls_handshake.py | 1 + ooni/nettests/manipulation/captiveportal.py | 2 + ooni/nettests/manipulation/daphne.py | 3 +- ooni/nettests/manipulation/dnsspoof.py | 2 + .../manipulation/http_header_field_manipulation.py | 2 + ooni/nettests/manipulation/http_host.py | 2 + .../manipulation/http_invalid_request_line.py | 2 + ooni/nettests/manipulation/traceroute.py | 5 ++ ooni/nettests/scanning/http_url_list.py | 5 +- ooni/nettests/third_party/netalyzr.py | 2 + ooni/oonicli.py | 9 +++- 27 files changed, 73 insertions(+), 55 deletions(-)
diff --git a/ooni/deck.py b/ooni/deck.py index 47edadc..cfd5d73 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -86,6 +86,7 @@ class Deck(InputFile): deckFile=None, decks_directory=config.decks_directory): self.id = deck_hash + self.requiresTor = False self.bouncer = '' self.netTestLoaders = [] self.inputs = [] @@ -135,12 +136,15 @@ class Deck(InputFile): return False try: net_test_loader.checkOptions() + if net_test_loader.requiresTor: + self.requiresTor = True except e.MissingRequiredOption, missing_options: if not self.bouncer: raise for missing_option in missing_options.message: if not has_test_helper(missing_option): raise + self.requiresTor = True self.netTestLoaders.append(net_test_loader)
@defer.inlineCallbacks diff --git a/ooni/director.py b/ooni/director.py index e4fc261..16c2069 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -118,10 +118,10 @@ class Director(object): return nettests
@defer.inlineCallbacks - def start(self): + def start(self, start_tor=False): self.netTests = self.getNetTests()
- if config.advanced.start_tor: + if config.advanced.start_tor and start_tor: yield self.startTor() elif config.tor.control_port: log.msg("Connecting to Tor Control Port...") diff --git a/ooni/nettest.py b/ooni/nettest.py index 61b327e..8ac9340 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -38,44 +38,6 @@ def get_test_methods(item, method_prefix="test_"): pass return test_cases
-def loadNetTestString(net_test_string): - """ - Load NetTest from a string. - WARNING input to this function *MUST* be sanitized and *NEVER* be - untrusted. - Failure to do so will result in code exec. - - net_test_string: - - a string that contains the net test to be run. - """ - net_test_file_object = StringIO(net_test_string) - - ns = {} - test_cases = [] - exec net_test_file_object.read() in ns - for item in ns.itervalues(): - test_cases.extend(get_test_methods(item)) - - if not test_cases: - raise e.NoTestCasesFound - - return test_cases - -def loadNetTestFile(net_test_file): - """ - Load NetTest from a file. - """ - test_cases = [] - module = filenameToModule(net_test_file) - for __, item in getmembers(module): - test_cases.extend(get_test_methods(item)) - - if not test_cases: - raise e.NoTestCasesFound - - return test_cases - def getTestClassFromFile(net_test_file): """ Will return the first class that is an instance of NetTestCase. @@ -173,20 +135,18 @@ def getNetTestInformation(net_test_file): class NetTestLoader(object): method_prefix = 'test' collector = None + requiresTor = False
def __init__(self, options, test_file=None, test_string=None): self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}.onion)/input/([a-z0-9]{64})$") self.options = options - self.testCases, test_cases = None, None + self.testCases = []
if test_file: - test_cases = loadNetTestFile(test_file) + self.loadNetTestFile(test_file) elif test_string: - test_cases = loadNetTestString(test_string) + self.loadNetTestString(test_string)
- if test_cases: - self.setupTestCases(test_cases) - @property def requiredTestHelpers(self): required_test_helpers = [] @@ -410,6 +370,8 @@ class NetTestLoader(object): test_instance = klass() if test_instance.requiresRoot: checkForRoot() + if test_instance.requiresTor: + self.requiresTor = True test_instance._checkRequiredOptions() test_instance._checkValidOptions()
@@ -671,6 +633,7 @@ class NetTestCase(object): requiredTestHelpers = {} requiredOptions = [] requiresRoot = False + requiresTor = False
localOptions = {} def _setUp(self): diff --git a/ooni/nettests/blocking/dnsconsistency.py b/ooni/nettests/blocking/dnsconsistency.py index 49e5360..5d17741 100644 --- a/ooni/nettests/blocking/dnsconsistency.py +++ b/ooni/nettests/blocking/dnsconsistency.py @@ -46,6 +46,8 @@ class DNSConsistencyTest(dnst.DNSTest): 'Input file of list of hostnames to attempt to resolve']
requiredTestHelpers = {'backend': 'dns'} + requiresRoot = False + requiresTor = False
usageOptions = UsageOptions requiredOptions = ['backend', 'file'] diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index 17859f5..84c80dc 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -36,6 +36,8 @@ class HTTPRequestsTest(httpt.HTTPTest):
inputFile = ['file', 'f', None, 'List of URLS to perform GET and POST requests to'] + requiresRoot = False + requiresTor = False
# These values are used for determining censorship based on response body # lengths diff --git a/ooni/nettests/blocking/tcp_connect.py b/ooni/nettests/blocking/tcp_connect.py index 129a2d5..84ceb2d 100644 --- a/ooni/nettests/blocking/tcp_connect.py +++ b/ooni/nettests/blocking/tcp_connect.py @@ -20,7 +20,9 @@ class TCPConnectTest(nettest.NetTestCase): version = "0.1" inputFile = ['file', 'f', None, 'File containing the IP:PORT combinations to be tested, one per line'] - + + requiresTor = False + requiresRoot = False requiredOptions = ['file'] def test_connect(self): """ diff --git a/ooni/nettests/experimental/dns_injection.py b/ooni/nettests/experimental/dns_injection.py index 97233cf..cf25539 100644 --- a/ooni/nettests/experimental/dns_injection.py +++ b/ooni/nettests/experimental/dns_injection.py @@ -30,6 +30,8 @@ class DNSInjectionTest(dnst.DNSTest):
usageOptions = UsageOptions requiredOptions = ['resolver', 'file'] + requiresRoot = False + requiresTor = False
def setUp(self): self.resolver = (self.localOptions['resolver'], 53) diff --git a/ooni/nettests/experimental/domclass_collector.py b/ooni/nettests/experimental/domclass_collector.py index c1866f2..6384e62 100644 --- a/ooni/nettests/experimental/domclass_collector.py +++ b/ooni/nettests/experimental/domclass_collector.py @@ -20,6 +20,8 @@ class DOMClassCollector(httpt.HTTPTest): followRedirects = True
inputFile = ['file', 'f', None, 'The list of urls to build a domclass for'] + requiresTor = False + requiresRoot = False
def test_collect(self): if self.input: diff --git a/ooni/nettests/experimental/http_filtering_bypassing.py b/ooni/nettests/experimental/http_filtering_bypassing.py index dc103db..1faed37 100644 --- a/ooni/nettests/experimental/http_filtering_bypassing.py +++ b/ooni/nettests/experimental/http_filtering_bypassing.py @@ -20,10 +20,13 @@ class HTTPFilteringBypass(tcpt.TCPTest):
usageOptions = UsageOptions requiredOptions = ['backend'] + requiresRoot = False + requiresTor = False
def setUp(self): self.port = int(self.localOptions['backendport']) self.address = self.localOptions['backend'] + self.report['tampering'] = None
def check_for_manipulation(self, response, payload): log.debug("Checking if %s == %s" % (response, payload)) diff --git a/ooni/nettests/experimental/http_keyword_filtering.py b/ooni/nettests/experimental/http_keyword_filtering.py index 0ae9c52..cbf12d1 100644 --- a/ooni/nettests/experimental/http_keyword_filtering.py +++ b/ooni/nettests/experimental/http_keyword_filtering.py @@ -26,6 +26,8 @@ class HTTPKeywordFiltering(httpt.HTTPTest): inputFile = ['file', 'f', None, 'List of keywords to use for censorship testing']
usageOptions = UsageOptions + requiresTor = False + requiresRoot = False
requiredOptions = ['backend']
diff --git a/ooni/nettests/experimental/http_trix.py b/ooni/nettests/experimental/http_trix.py index 85a4ba2..b5c9ae2 100644 --- a/ooni/nettests/experimental/http_trix.py +++ b/ooni/nettests/experimental/http_trix.py @@ -16,6 +16,8 @@ class HTTPTrix(tcpt.TCPTest): authors = "Arturo Filastò"
usageOptions = UsageOptions + requiresTor = False + requiresRoot = False requiredOptions = ['backend']
def setUp(self): diff --git a/ooni/nettests/experimental/http_uk_mobile_networks.py b/ooni/nettests/experimental/http_uk_mobile_networks.py index 784a9e9..cc00fdc 100644 --- a/ooni/nettests/experimental/http_uk_mobile_networks.py +++ b/ooni/nettests/experimental/http_uk_mobile_networks.py @@ -33,6 +33,8 @@ class HTTPUKMobileNetworksTest(httpt.HTTPTest):
inputFile = ['urls', 'f', None, 'List of urls one per line to test for censorship'] requiredOptions = ['urls'] + requiresRoot = False + requiresTor = False
def testPattern(self, value, pattern, type): if type == 'eq': @@ -80,6 +82,3 @@ class HTTPUKMobileNetworksTest(httpt.HTTPTest): log.msg("Testing rules %s" % rules) redirect = self.testRules(rules, location) self.report['redirect'] = redirect - - - diff --git a/ooni/nettests/experimental/keyword_filtering.py b/ooni/nettests/experimental/keyword_filtering.py index 9eec4ff..0937efd 100644 --- a/ooni/nettests/experimental/keyword_filtering.py +++ b/ooni/nettests/experimental/keyword_filtering.py @@ -26,6 +26,8 @@ class KeywordFiltering(scapyt.BaseScapyTest):
inputFile = ['file', 'f', None, 'List of keywords to use for censorship testing'] + requiresRoot = True + requiresTor = False
def test_tcp_keyword_filtering(self): """ diff --git a/ooni/nettests/experimental/parasitictraceroute.py b/ooni/nettests/experimental/parasitictraceroute.py index c8aa3ed..9526455 100644 --- a/ooni/nettests/experimental/parasitictraceroute.py +++ b/ooni/nettests/experimental/parasitictraceroute.py @@ -13,6 +13,8 @@ class ParasiticTracerouteTest(scapyt.BaseScapyTest): version = '0.1'
samplePeriod = 40 + requiresTor = False + requiresRoot = False
def setUp(self): self.report['parasitic_traceroute'] = {} diff --git a/ooni/nettests/experimental/script.py b/ooni/nettests/experimental/script.py index 4772f65..c03077a 100644 --- a/ooni/nettests/experimental/script.py +++ b/ooni/nettests/experimental/script.py @@ -66,6 +66,8 @@ class Script(nettest.NetTestCase):
usageOptions = UsageOptions requiredOptions = ['interpreter', 'script'] + requiresRoot = False + requiresTor = False
def test_run_script(self): """ diff --git a/ooni/nettests/experimental/squid.py b/ooni/nettests/experimental/squid.py index 777bc3e..17255e5 100644 --- a/ooni/nettests/experimental/squid.py +++ b/ooni/nettests/experimental/squid.py @@ -23,6 +23,10 @@ class SquidTest(httpt.HTTPTest):
#inputFile = ['urls', 'f', None, 'Urls file'] inputs =['http://google.com'] + + requiresRoot = False + requiresTor = False + def test_cacheobject(self): """ This detects the presence of a squid transparent HTTP proxy by sending diff --git a/ooni/nettests/experimental/tls_handshake.py b/ooni/nettests/experimental/tls_handshake.py index f775935..a38983a 100644 --- a/ooni/nettests/experimental/tls_handshake.py +++ b/ooni/nettests/experimental/tls_handshake.py @@ -143,6 +143,7 @@ class HandshakeTest(nettest.NetTestCase): version = '0.0.3'
requiresRoot = False + requiresTor = False usageOptions = HandshakeOptions
host = None diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index afbdfeb..d6d8040 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -72,6 +72,8 @@ class CaptivePortal(nettest.NetTestCase): version = '0.2' author = "Isis Lovecruft" usageOptions = UsageOptions + requiresRoot = False + requiresTor = False
def http_fetch(self, url, headers={}): """ diff --git a/ooni/nettests/manipulation/daphne.py b/ooni/nettests/manipulation/daphne.py index 936111a..545514b 100644 --- a/ooni/nettests/manipulation/daphne.py +++ b/ooni/nettests/manipulation/daphne.py @@ -69,7 +69,8 @@ class daphn3Test(nettest.NetTestCase): 'Specify the pcap or YAML file to be used as input to the test']
#requiredOptions = ['file'] - + requiresRoot = False + requiresTor = False steps = None
def inputProcessor(self, filename): diff --git a/ooni/nettests/manipulation/dnsspoof.py b/ooni/nettests/manipulation/dnsspoof.py index 82fb325..15aad07 100644 --- a/ooni/nettests/manipulation/dnsspoof.py +++ b/ooni/nettests/manipulation/dnsspoof.py @@ -32,6 +32,8 @@ class DNSSpoof(scapyt.ScapyTest):
requiredTestHelpers = {'backend': 'dns'} requiredOptions = ['hostname', 'resolver'] + requiresRoot = True + requiresTor = False
def setUp(self): self.resolverAddr, self.resolverPort = self.localOptions['resolver'].split(':') diff --git a/ooni/nettests/manipulation/http_header_field_manipulation.py b/ooni/nettests/manipulation/http_header_field_manipulation.py index 93beb25..3f1b4ed 100644 --- a/ooni/nettests/manipulation/http_header_field_manipulation.py +++ b/ooni/nettests/manipulation/http_header_field_manipulation.py @@ -51,6 +51,8 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest):
requiredTestHelpers = {'backend': 'http-return-json-headers'} requiredOptions = ['backend'] + requiresTor = False + requiresRoot = False
def get_headers(self): headers = {} diff --git a/ooni/nettests/manipulation/http_host.py b/ooni/nettests/manipulation/http_host.py index 5e77415..7207c10 100644 --- a/ooni/nettests/manipulation/http_host.py +++ b/ooni/nettests/manipulation/http_host.py @@ -47,6 +47,8 @@ class HTTPHost(httpt.HTTPTest):
requiredTestHelpers = {'backend': 'http-return-json-headers'} requiredOptions = ['backend'] + requiresTor = False + requiresRoot = False
def setUp(self): self.report['transparent_http_proxy'] = False diff --git a/ooni/nettests/manipulation/http_invalid_request_line.py b/ooni/nettests/manipulation/http_invalid_request_line.py index 2812e45..6769269 100644 --- a/ooni/nettests/manipulation/http_invalid_request_line.py +++ b/ooni/nettests/manipulation/http_invalid_request_line.py @@ -28,6 +28,8 @@ class HTTPInvalidRequestLine(tcpt.TCPTest):
requiredTestHelpers = {'backend': 'tcp-echo'} requiredOptions = ['backend'] + requiresRoot = False + requiresTor = False
def setUp(self): self.port = int(self.localOptions['backendport']) diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index e74c864..66e0a34 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -24,8 +24,13 @@ class UsageOptions(usage.Options):
class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" + description = "Performs a UDP, TCP, ICMP traceroute with destination port number set to 0, 22, 23, 53, 80, 123, 443, 8080 and 65535" + requiredTestHelpers = {'backend': 'traceroute'} + requiresRoot = True + requiresTor = False + usageOptions = UsageOptions dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535] version = "0.3" diff --git a/ooni/nettests/scanning/http_url_list.py b/ooni/nettests/scanning/http_url_list.py index 0accaae..98e4ba0 100644 --- a/ooni/nettests/scanning/http_url_list.py +++ b/ooni/nettests/scanning/http_url_list.py @@ -29,6 +29,9 @@ class HTTPURLList(httpt.HTTPTest):
usageOptions = UsageOptions
+ requiresRoot = False + requiresTor = False + inputFile = ['file', 'f', None, 'List of URLS to perform GET and POST requests to']
@@ -94,5 +97,3 @@ class HTTPURLList(httpt.HTTPTest):
def test_put(self): return self.doRequest(self.url, method="PUT") - - diff --git a/ooni/nettests/third_party/netalyzr.py b/ooni/nettests/third_party/netalyzr.py index 9b21831..63ab3c7 100644 --- a/ooni/nettests/third_party/netalyzr.py +++ b/ooni/nettests/third_party/netalyzr.py @@ -14,6 +14,8 @@ from twisted.internet import reactor, threads, defer
class NetalyzrWrapperTest(nettest.NetTestCase): name = "NetalyzrWrapper" + requiresRoot = False + requiresTor = False
def setUp(self): cwd = os.path.abspath(os.path.join(os.path.abspath(__file__), '..')) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 935d091..ed7b297 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -133,8 +133,6 @@ def runWithDirector(logging=True, start_tor=True):
sys.exit(0)
- d = director.start() - #XXX: This should mean no bouncer either! if global_options['no-collector']: log.msg("Not reporting using a collector") @@ -143,6 +141,11 @@ def runWithDirector(logging=True, start_tor=True):
deck = Deck() deck.bouncer = global_options['bouncer'] + start_tor = deck.requiresTor + if global_options['bouncer']: + start_tor = True + if global_options['collector']: + start_tor = True
try: if global_options['testdeck']: @@ -165,6 +168,8 @@ def runWithDirector(logging=True, start_tor=True): print net_test_loader.usageOptions().getUsage() sys.exit(2)
+ d = director.start(start_tor=start_tor) + def setup_nettest(_): try: return deck.setup()