- tor-commits - lists.torproject.org

[ooni-probe/master] Fix the last conflicts, hopefully... argh...
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 793f013cf2ca50f415398677dd12ab59ae9c57b4 Merge: 45cc771 428e8e8 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 16:41:48 2012 +0200 Fix the last conflicts, hopefully... argh... Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe Conflicts: ooni/plugins/captiveportal.py ooni/plugins/captiveportal.py | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-)

1 0

[metrics-tasks/master] Fail gracefully for old consensus
by karsten＠torproject.org 09 Jul '12

09 Jul '12

commit cc86e841928a7a85168143ff524b0779c0e5059a Author: Sathyanarayanan <gsathya.ceg(a)gmail.com> Date: Mon Jul 9 16:57:08 2012 +0300 Fail gracefully for old consensus --- task-6232/pyentropy.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 0ae0f25..23850d1 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -61,8 +61,8 @@ def run(file_name): totalExitBW += router.bandwidth if len(routers) <= 0: - print "Error: amount of routers must be > 0." - return; + print "Error: Old consensus file. Not able to parse." + return entropy, entropy_exit, entropy_guard = 0.0, 0.0, 0.0 for router in routers: @@ -92,4 +92,5 @@ if __name__ == "__main__": with open(sys.argv[2], 'w') as f: for file_name in os.listdir(sys.argv[1]): string = run(os.path.join(sys.argv[1], file_name)) - f.write("%s\n" % (string)) + if string: + f.write("%s\n" % (string))

1 0

[metrics-tasks/master] Merge remote-tracking branch 'gsathya/master'
by karsten＠torproject.org 09 Jul '12

09 Jul '12

commit b2d9e509b4c513aa2e253a550fd832fb9c86a94e Merge: b939e76 cc86e84 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 9 16:16:24 2012 +0200 Merge remote-tracking branch 'gsathya/master' task-6232/pyentropy.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-)

1 0

[metrics-tasks/master] Take out noisy warning.
by karsten＠torproject.org 09 Jul '12

09 Jul '12

commit 3d4e484df398a5fd395149f1fb87b3b702dd4a2c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 9 16:16:44 2012 +0200 Take out noisy warning. --- task-6232/pyentropy.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 23850d1..650423b 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -61,7 +61,6 @@ def run(file_name): totalExitBW += router.bandwidth if len(routers) <= 0: - print "Error: Old consensus file. Not able to parse." return entropy, entropy_exit, entropy_guard = 0.0, 0.0, 0.0

1 0

[ooni-probe/master] Remove dead code.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit bcf5e79efcc8273a424a6bc668e0b5d6b21f0b0a Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 12:01:12 2012 +0200 Remove dead code. --- ooni/plugoo/work.py | 81 --------------------------------------------------- 1 files changed, 0 insertions(+), 81 deletions(-) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index a3fb168..fc338d7 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -107,84 +107,3 @@ class WorkGenerator(object): self.idx += 1 return (ret, self.Test, self.idx) - def p_next(self): - """ - XXX This is not used for KISS sake. - """ - if self.end: - raise StopIteration - - if not self.assetGenerator: - self.end = True - return WorkUnit(None, self.assetNames, self.Test, self.idx) - - # Plank asset - p_asset = [] - for i in xrange(0, self.size): - try: - asset = self.assetGenerator.next() - p_asset.append(asset) - print p_asset - except StopIteration: - if self.asset_num > 1: - pass - self.end = True - break - - self.idx += 1 - return WorkUnit(p_asset, self.assetNames, self.Test, self.idx) - -class WorkUnit(object): - """ - XXX This is currently not implemented for KISS sake. - - This is an object responsible for completing WorkUnits it will - return its result in a deferred. - - The execution of a unit of work should be Atomic. - - Reporting to the OONI-net happens on completion of a Unit of Work. - - @Node node: This represents the node associated with the Work Unit - @Asset asset: This is the asset associated with the Work Unit - @Test test: This represents the Test to be with the specified assets - @ivar arguments: These are the extra attributes to be passsed to the Test - """ - - node = None - asset = None - test = None - arguments = None - - def __init__(self, asset, assetNames, test, idx): - self.asset = asset - if not asset: - self.assetGenerator = iter([1]) - else: - self.assetGenerator = iter(asset) - self.Test = test - self.assetNames = assetNames - self.idx = idx - - def __iter__(self): - return self - - def __repr__(self): - return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx) - - def serialize(self): - """ - Serialize this unit of work for RPC activity. - """ - return yaml.dump(self) - - def next(self): - """ - Launches the Unit of Work with the specified assets on the node. - """ - try: - asset = self.assetGenerator.next() - ret = self.Test.set_asset(asset) - return ret - except StopIteration: - raise StopIteration

1 0

[ooni-probe/master] Port Isis's captive portal test to the new framework.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 428e8e827e6f457aaf987be753b4f99c5127f750 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 12:41:49 2012 +0200 Port Isis's captive portal test to the new framework. --- ooni/plugins/captiveportal.py | 625 +++++++++++++++++++++++++++++++++++++++++ 1 files changed, 625 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py new file mode 100644 index 0000000..9b7bee3 --- /dev/null +++ b/ooni/plugins/captiveportal.py @@ -0,0 +1,625 @@ +# -*- coding: utf-8 -*- +""" + captiveportal + ************* + + This test is a collection of tests to detect the presence of a + captive portal. Code is taken, in part, from the old ooni-probe, + which was written by Jacob Appelbaum and Arturo Filastò. + + :copyright: (c) 2012 Isis Lovecruft + :license: see LICENSE for more details +""" +import base64 +import os +import random +import re +import string +import urllib2 +from urlparse import urlparse + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest +from ooni.protocols import http +from ooni import log + +try: + from dns import resolver +except ImportError: + print "The dnspython module was not found. https://crate.io/packages/dnspython/" + +__plugoo__ = "captiveportal" +__desc__ = "Captive portal detection test" + +class CaptivePortalArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'], + ['user-agent', 'u', random.choice(http.useragents), + 'User agent for HTTP requests'] + ] + +class CaptivePortal(OONITest): + """ + Compares content and status codes of HTTP responses, and attempts + to determine if content has been altered. + """ + + implements(IPlugin, ITest) + + shortName = "captivep" + description = "Captive Portal Test" + requirements = None + options = CaptivePortalArgs + # Tells this to be blocking. + blocking = True + + def http_fetch(self, url, headers={}): + """ + Parses an HTTP url, fetches it, and returns a urllib2 response + object. + """ + url = urlparse(url).geturl() + request = urllib2.Request(url, None, headers) + response = urllib2.urlopen(request) + response_headers = dict(response.headers) + return response, response_headers + + def http_content_match_fuzzy_opt(self, experimental_url, control_result, + headers=None, fuzzy=False): + """ + Makes an HTTP request on port 80 for experimental_url, then + compares the response_content of experimental_url with the + control_result. Optionally, if the fuzzy parameter is set to + True, the response_content is compared with a regex of the + control_result. If the response_content from the + experimental_url and the control_result match, returns True + with the HTTP status code and headers; False, status code, and + headers if otherwise. + """ + + if headers is None: + default_ua = self.local_options['user-agent'] + headers = {'User-Agent': default_ua} + + response, response_headers = self.http_fetch(experimental_url, headers) + response_content = response.read() + response_code = response.code + if response_content is not None: + if fuzzy: + pattern = re.compile(control_result) + match = pattern.search(response_content) + if not match: + log.msg("Fuzzy HTTP content comparison for experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("does not match!") + return False, response_code, response_headers + else: + log.msg("Fuzzy HTTP content comparison of experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("and the expected control result yielded a match.") + return True, response_code, response_headers + else: + if str(response_content) != str(control_result): + log.msg("HTTP content comparison of experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("and the expected control result do not match.") + return False, response_code, response_headers + else: + return True, response_code, response_headers + else: + log.warn("HTTP connection appears to have failed.") + return False, False, False + + def http_status_code_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they match. + """ + if int(experiment_code) != int(control_code): + return False + return True + + def http_status_code_no_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they do not match. + """ + if self.http_status_code_match(experiment_code, control_code): + return False + return True + + def dns_resolve(self, hostname, nameserver=None): + """ + Resolves hostname(s) though nameserver to corresponding + address(es). hostname may be either a single hostname string, + or a list of strings. If nameserver is not given, use local + DNS resolver, and if that fails try using 8.8.8.8. + """ + + if isinstance(hostname, str): + hostname = [hostname] + + if nameserver is not None: + res = resolver.Resolver(configure=False) + res.nameservers = [nameserver] + else: + res = resolver.Resolver() + + response = [] + answer = None + + for hn in hostname: + try: + answer = res.query(hn) + except resolver.NoNameservers: + res.nameservers = ['8.8.8.8'] + try: + answer = res.query(hn) + except resolver.NXDOMAIN: + log.msg("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + except resolver.NXDOMAIN: + log.msg("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + finally: + if answer: + for addr in answer: + response.append(addr.address) + return response + + def dns_resolve_match(self, experiment_hostname, control_address): + """ + Resolve experiment_hostname, and check to see that it returns + an experiment_address which matches the control_address. If + they match, returns True and experiment_address; otherwise + returns False and experiment_address. + """ + experiment_address = self.dns_resolve(experiment_hostname) + if experiment_address: + if len(set(experiment_address) & set([control_address])) > 0: + return True, experiment_address + else: + log.msg("DNS comparison of control '%s' does not" % control_address) + log.msg("match experiment response '%s'" % experiment_address) + return False, experiment_address + else: + log.debug("dns_resolve() for %s failed" % experiment_hostname) + return None, experiment_address + + def get_auth_nameservers(self, hostname): + """ + Many CPs set a nameserver to be used. Let's query that + nameserver for the authoritative nameservers of hostname. + + The equivalent of: + $ dig +short NS ooni.nu + """ + res = resolver.Resolver() + answer = res.query(hostname, 'NS') + auth_nameservers = [] + for auth in answer: + auth_nameservers.append(auth.to_text()) + return auth_nameservers + + def hostname_to_0x20(self, hostname): + """ + MaKEs yOur HOsTnaME lOoK LiKE THis. + + For more information, see: + D. Dagon, et. al. "Increased DNS Forgery Resistance + Through 0x20-Bit Encoding". Proc. CSS, 2008. + """ + hostname_0x20 = '' + for char in hostname: + l33t = random.choice(['caps', 'nocaps']) + if l33t == 'caps': + hostname_0x20 = hostname_0x20 + char.capitalize() + else: + hostname_0x20 = hostname_0x20 + char.lower() + return hostname_0x20 + + def check_0x20_to_auth_ns(self, hostname, sample_size=None): + """ + Resolve a 0x20 DNS request for hostname over hostname's + authoritative nameserver(s), and check to make sure that + the capitalization in the 0x20 request matches that of the + response. Also, check the serial numbers of the SOA (Start + of Authority) records on the authoritative nameservers to + make sure that they match. + + If sample_size is given, a random sample equal to that number + of authoritative nameservers will be queried; default is 5. + """ + log.msg("") + log.msg("Testing random capitalization of DNS queries...") + log.msg("Testing that Start of Authority serial numbers match...") + + auth_nameservers = self.get_auth_nameservers(hostname) + + if sample_size is None: + sample_size = 5 + resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), + sample_size) + + querynames = [] + answernames = [] + serials = [] + + # Even when gevent monkey patching is on, the requests here + # are sent without being 0x20'd, so we need to 0x20 them. + hostname = self.hostname_to_0x20(hostname) + + for auth_ns in resolved_auth_ns: + res = resolver.Resolver(configure=False) + res.nameservers = [auth_ns] + try: + answer = res.query(hostname, 'SOA') + except resolver.Timeout: + continue + querynames.append(answer.qname.to_text()) + answernames.append(answer.rrset.name.to_text()) + for soa in answer: + serials.append(str(soa.serial)) + + if len(set(querynames).intersection(answernames)) == 1: + log.msg("Capitalization in DNS queries and responses match.") + name_match = True + else: + log.msg("The random capitalization '%s' used in" % hostname) + log.msg("DNS queries to that hostname's authoritative") + log.msg("nameservers does not match the capitalization in") + log.msg("the response.") + name_match = False + + if len(set(serials)) == 1: + log.msg("Start of Authority serial numbers all match.") + serial_match = True + else: + log.msg("Some SOA serial numbers did not match the rest!") + serial_match = False + + ret = name_match, serial_match, querynames, answernames, serials + + if name_match and serial_match: + log.msg("Your DNS queries do not appear to be tampered.") + return ret + elif name_match or serial_match: + log.msg("Something is tampering with your DNS queries.") + return ret + elif not name_match and not serial_match: + log.msg("Your DNS queries are definitely being tampered with.") + return ret + + def get_random_url_safe_string(self, length): + """ + Returns a random url-safe string of specified length, where + 0 < length <= 256. The returned string will always start with + an alphabetic character. + """ + if (length <= 0): + length = 1 + elif (length > 256): + length = 256 + + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + while not random_ascii[:1].isalpha(): + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + three_quarters = int((len(random_ascii)) * (3.0/4.0)) + random_string = random_ascii[:three_quarters] + return random_string + + def get_random_hostname(self, length=None): + """ + Returns a random hostname with SLD of specified length. If + length is unspecified, length=32 is used. + + These *should* all resolve to NXDOMAIN. If they actually + resolve to a box that isn't part of a captive portal that + would be rather interesting. + """ + if length is None: + length = 32 + + random_sld = self.get_random_url_safe_string(length) + + # if it doesn't start with a letter, chuck it. + while not random_sld[:1].isalpha(): + random_sld = self.get_random_url_safe_string(length) + + tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] + random_tld = urllib2.random.choice(tld_list) + random_hostname = random_sld + random_tld + return random_hostname + + def compare_random_hostnames(self, hostname_count=None, hostname_length=None): + """ + Get hostname_count number of random hostnames with SLD length + of hostname_length, and then attempt DNS resolution. If no + arguments are given, default to three hostnames of 32 bytes + each. These random hostnames *should* resolve to NXDOMAIN, + except in the case where a user is presented with a captive + portal and remains unauthenticated, in which case the captive + portal may return the address of the authentication page. + + If the cardinality of the intersection of the set of resolved + random hostnames and the single element control set + (['NXDOMAIN']) are equal to one, then DNS properly resolved. + + Returns true if only NXDOMAINs were returned, otherwise returns + False with the relative complement of the control set in the + response set. + """ + if hostname_count is None: + hostname_count = 3 + + log.msg("Generating random hostnames...") + log.msg("Resolving DNS for %d random hostnames..." % hostname_count) + + control = ['NXDOMAIN'] + responses = [] + + for x in range(hostname_count): + random_hostname = self.get_random_hostname(hostname_length) + response_match, response_address = self.dns_resolve_match(random_hostname, + control[0]) + for address in response_address: + if response_match is False: + log.msg("Strangely, DNS resolution of the random hostname") + log.msg("%s actually points to %s" + % (random_hostname, response_address)) + responses = responses + [address] + else: + responses = responses + [address] + + intersection = set(responses) & set(control) + relative_complement = set(responses) - set(control) + r = set(responses) + + if len(intersection) == 1: + log.msg("All %d random hostnames properly resolved to NXDOMAIN." + % hostname_count) + return True, relative_complement + elif (len(intersection) == 1) and (len(r) > 1): + log.msg("Something odd happened. Some random hostnames correctly") + log.msg("resolved to NXDOMAIN, but several others resolved to") + log.msg("to the following addresses: %s" % relative_complement) + return False, relative_complement + elif (len(intersection) == 0) and (len(r) == 1): + log.msg("All random hostnames resolved to the IP address ") + log.msg("'%s', which is indicative of a captive portal." % r) + return False, relative_complement + else: + log.debug("Apparently, pigs are flying on your network, 'cause a") + log.debug("bunch of hostnames made from 32-byte random strings") + log.debug("just magically resolved to a bunch of random addresses.") + log.debug("That is definitely highly improbable. In fact, my napkin") + log.debug("tells me that the probability of just one of those") + log.degug("hostnames resolving to an address is 1.68e-59, making") + log.debug("it nearly twice as unlikely as an MD5 hash collision.") + log.debug("Either someone is seriously messing with your network,") + log.debug("or else you are witnessing the impossible. %s" % r) + return False, relative_complement + + def google_dns_cp_test(self): + """ + Google Chrome resolves three 10-byte random hostnames. + """ + subtest = "Google Chrome DNS-based" + + log.msg("") + log.msg("Running the Google Chrome DNS-based captive portal test...") + + gmatch, google_dns_result = self.compare_random_hostnames(3, 10) + + if gmatch: + log.msg("Google Chrome DNS-based captive portal test did not") + log.msg("detect a captive portal.") + return google_dns_result + else: + log.msg("Google Chrome DNS-based captive portal test believes") + log.msg("you are in a captive portal, or else something very") + log.msg("odd is happening with your DNS.") + return google_dns_result + + def ms_dns_cp_test(self): + """ + Microsoft "phones home" to a server which will always resolve + to the same address. + """ + subtest = "Microsoft NCSI DNS-based" + + log.msg("") + log.msg("Running the Microsoft NCSI DNS-based captive portal") + log.msg("test...") + + msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", + "131.107.255.255") + if msmatch: + log.msg("Microsoft NCSI DNS-based captive portal test did not") + log.msg("detect a captive portal.") + return ms_dns_result + else: + log.msg("Microsoft NCSI DNS-based captive portal test ") + log.msg("believes you are in a captive portal.") + return ms_dns_result + + def run_vendor_dns_tests(self): + """ + Run the vendor DNS tests. + """ + report = {} + report['google_dns_cp'] = self.google_dns_cp_test() + report['ms_dns_cp'] = self.ms_dns_cp_test() + + return report + + def run_vendor_tests(self, *a, **kw): + """ + These are several vendor tests used to detect the presence of + a captive portal. Each test compares HTTP status code and + content to the control results and has its own User-Agent + string, in order to emulate the test as it would occur on the + device it was intended for. Vendor tests are defined in the + format: + [exp_url, ctrl_result, ctrl_code, ua, test_name] + """ + + vendor_tests = [['http://www.apple.com/library/test/success.html', + 'Success', + '200', + 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3', + 'Apple HTTP Captive Portal'], + ['http://tools.ietf.org/html/draft-nottingham-http-portal-02', + '428 Network Authentication Required', + '428', + 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0', + 'W3 Captive Portal'], + ['http://www.msftncsi.com/ncsi.txt', + 'Microsoft NCSI', + '200', + 'Microsoft NCSI', + 'MS HTTP Captive Portal',]] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + def compare_content(status_func, fuzzy, experiment_url, control_result, + control_code, headers, test_name): + log.msg("") + log.msg("Running the %s test..." % test_name) + + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + headers, fuzzy) + status_match = status_func(experiment_code, control_code) + + if status_match and content_match: + log.msg("The %s test was unable to detect" % test_name) + log.msg("a captive portal.") + return True + else: + log.msg("The %s test shows that your network" % test_name) + log.msg("is filtered.") + return False + + result = [] + for vt in vendor_tests: + report = {} + report['vt'] = vt + + experiment_url = vt[0] + control_result = vt[1] + control_code = vt[2] + headers = {'User-Agent': vt[3]} + test_name = vt[4] + + args = (experiment_url, control_result, control_code, headers, test_name) + + if test_name == "MS HTTP Captive Portal": + report['result'] = compare_content(sm, False, *args) + + elif test_name == "Apple HTTP Captive Portal": + report['result'] = compare_content(sm, True, *args) + + elif test_name == "W3 Captive Portal": + report['result'] = compare_content(snm, True, *args) + + else: + log.warn("Ooni is trying to run an undefined CP vendor test.") + result.append(report) + return result + + def control(self, experiment_result, args): + """ + Compares the content and status code of the HTTP response for + experiment_url with the control_result and control_code + respectively. If the status codes match, but the experimental + content and control_result do not match, fuzzy matching is enabled + to determine if the control_result is at least included somewhere + in the experimental content. Returns True if matches are found, + and False if otherwise. + """ + experiment_url = self.local_options['experiment-url'] + control_result = 'XX' + control_code = 200 + ua = self.local_options['user-agent'] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + log.msg("Running test for '%s'..." % experiment_url) + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result) + status_match = sm(experiment_code, control_code) + if status_match and content_match: + log.msg("The test for '%s'" % experiment_url) + log.msg("was unable to detect a captive portal.") + return experiment_result, True + + elif status_match and not content_match: + log.msg("Retrying '%s' with fuzzy match enabled." + % experiment_url) + fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + fuzzy=True) + if fuzzy_match: + return experiment_result, True + else: + log.msg("Found modified content on '%s'," % experiment_url) + log.msg("which could indicate a captive portal.") + + return experiment_result, False + else: + log.msg("The content comparison test for ") + log.msg("'%s'" % experiment_url) + log.msg("shows that your HTTP traffic is filtered.") + return experiment_result, False + + def experiment(self, args): + """ + Runs the CaptivePortal(Test). + + CONFIG OPTIONS + -------------- + + If "do_captive_portal_vendor_tests" is set to "true", then vendor + specific captive portal HTTP-based tests will be run. + + If "do_captive_portal_dns_tests" is set to "true", then vendor + specific captive portal DNS-based tests will be run. + + If "check_dns_requests" is set to "true", then Ooni-probe will + attempt to check that your DNS requests are not being tampered with + by a captive portal. + + If "captive_portal" = "yourfilename.txt", then user-specified tests + will be run. + + Any combination of the above tests can be run. + """ + report = {} + + log.msg("") + log.msg("Running vendor tests...") + report['vendor_tests'] = self.run_vendor_tests() + + log.msg("") + log.msg("Running vendor DNS-based tests...") + report['vendor_dns_tests'] = self.run_vendor_dns_tests() + + log.msg("") + log.msg("Checking that DNS requests are not being tampered...") + report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu') + + log.msg("") + log.msg("Captive portal test finished!") + return report + +cp = CaptivePortal(None, None, None)

1 0

[metrics-tasks/master] Merge remote-tracking branch 'gsathya/master'
by karsten＠torproject.org 09 Jul '12

09 Jul '12

commit b939e76767191ae28cf7d4308838e24e64c52ccf Merge: c4c6d16 9fb1eae Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 9 12:19:36 2012 +0200 Merge remote-tracking branch 'gsathya/master' task-6232/pyentropy.py | 64 ++++++++++++++++++----------------------------- 1 files changed, 25 insertions(+), 39 deletions(-)

1 0

[metrics-tasks/master] Revert back to original algorithm
by karsten＠torproject.org 09 Jul '12

09 Jul '12

commit 9fb1eaeb8c6104d99ed3d4d9f1748fa9269c6ac8 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Thu Jul 5 12:49:37 2012 +0200 Revert back to original algorithm Phw realised that the original algorithm made more sense in the current scenario where we need to find the probability of the node being selected and not the probability of the bandwidth being selected/present. Patched by phw --- task-6232/pyentropy.py | 64 ++++++++++++++++++----------------------------- 1 files changed, 25 insertions(+), 39 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 6fa5b86..0ae0f25 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -12,7 +12,6 @@ from decimal import * RESULTS = [] KEYS = ['r','s','v','w','p','m'] - class Router: def __init__(self): self.lines = [] @@ -22,7 +21,7 @@ class Router: self.probability = None self.is_exit = None self.is_guard = None - + def add(self, key, values): if key == 'r': self.nick = values[0] @@ -34,7 +33,7 @@ class Router: self.is_exit = True if "Guard" in self.flags: self.is_guard = True - + def run(file_name): routers = [] # parse consensus @@ -52,48 +51,35 @@ def run(file_name): valid_after = ' '.join(values) elif key in KEYS: router.add(key, values) - - # build hash table with freq. distribution - # key: bandwidth - # value: number of bandwidth's observations - - bw_dist, bw_dist_exit, bw_dist_guard = {}, {}, {} + + totalBW, totalExitBW, totalGuardBW = 0, 0, 0 for router in routers: - if router.is_exit: - if bw_dist_exit.has_key(router.bandwidth): - bw_dist_exit[router.bandwidth] += 1 - else: - bw_dist_exit[router.bandwidth] = 1 + totalBW += router.bandwidth if router.is_guard: - if bw_dist_guard.has_key(router.bandwidth): - bw_dist_guard[router.bandwidth] += 1 - else: - bw_dist_guard[router.bandwidth] = 1 - if bw_dist.has_key(router.bandwidth): - bw_dist[router.bandwidth] += 1 - else: - bw_dist[router.bandwidth] = 1 - + totalGuardBW += router.bandwidth + if router.is_exit: + totalExitBW += router.bandwidth + if len(routers) <= 0: print "Error: amount of routers must be > 0." return; - + entropy, entropy_exit, entropy_guard = 0.0, 0.0, 0.0 - for bw in bw_dist.iterkeys(): - # p = probability of one particular bandwidth - p = float(bw_dist[bw]) / len(routers) - entropy += -(p * math.log(p, 2)) - - for bw in bw_dist_exit.iterkeys(): - # p = probability of one particular bandwidth - p = float(bw_dist[bw]) / len(routers) - entropy_exit += -(p * math.log(p, 2)) - - for bw in bw_dist_guard.iterkeys(): - # p = probability of one particular bandwidth - p = float(bw_dist[bw]) / len(routers) - entropy_guard += -(p * math.log(p, 2)) - + for router in routers: + p = float(router.bandwidth) / float(totalBW) + if p != 0: + entropy += -(p * math.log(p, 2)) + + if router.is_guard: + p = float(router.bandwidth) / float(totalGuardBW) + if p != 0: + entropy_exit += -(p * math.log(p, 2)) + + if router.is_exit: + p = float(router.bandwidth) / float(totalExitBW) + if p != 0: + entropy_guard += -(p * math.log(p, 2)) + return ",".join([valid_after, str(entropy), str(entropy_exit), str(entropy_guard)]) def usage():

1 0

[ooni-probe/master] Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe into designimprove
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 500790c55d4742b41d680dc2f9f59e405c49ea66 Merge: d68e5f4 ce8e175 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 11:57:11 2012 +0200 Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe into designimprove ooni/plugoo/reports.py | 10 +++------- 1 files changed, 3 insertions(+), 7 deletions(-)

1 0

[ooni-probe/master] Do a big cleanup
by art＠torproject.org 09 Jul '12

09 Jul '12

commit d68e5f4f8edffbcd7e9c9fc5ce22030472f401a1 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 11:54:37 2012 +0200 Do a big cleanup * Better specify test writing interface * Move example plugins to appropriate directory * Implement RFC3339 timestamp, as spec * Make sure all the defaults tests run and output something even without arguments * Fix bug in worker --- ooni/date.py | 17 ++++- ooni/example_plugins/examplescapy.py | 49 ++++++++++++++ ooni/example_plugins/skel.py | 29 ++++++++ ooni/lib/Makefile | 8 ++- ooni/ooniprobe.py | 1 - ooni/plugins/blocking.py | 6 +- ooni/plugins/chinatrigger.py | 2 +- ooni/plugins/examplescapy.py | 49 -------------- ooni/plugins/httphost.py | 11 ++-- ooni/plugins/httpt.py | 5 +- ooni/plugins/skel.py | 29 -------- ooni/plugins/tcpconnect.py | 9 ++- ooni/plugoo/interface.py | 31 ++++++++- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 29 +++++--- ooni/plugoo/work.py | 118 ++++++++++++++++------------------ ooni/protocols/http.py | 5 +- 17 files changed, 223 insertions(+), 177 deletions(-) diff --git a/ooni/date.py b/ooni/date.py index 6f83191..59ec1f8 100644 --- a/ooni/date.py +++ b/ooni/date.py @@ -1,11 +1,22 @@ +from ooni.lib.rfc3339 import rfc3339 from datetime import datetime +class odate(datetime): + def __str__(self): + return rfc3339(self) + + def __repr__(self): + return "'%s'" % rfc3339(self) + + def from_rfc(self, datestr): + pass + +def now(): + return odate.utcnow() + def pretty_date(): cur_time = datetime.utcnow() d_format = "%d %B %Y %H:%M:%S" pretty = cur_time.strftime(d_format) return pretty -def now(): - return datetime.utcnow() - diff --git a/ooni/example_plugins/examplescapy.py b/ooni/example_plugins/examplescapy.py new file mode 100644 index 0000000..aa6d81b --- /dev/null +++ b/ooni/example_plugins/examplescapy.py @@ -0,0 +1,49 @@ +import random +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset +from ooni import log +from ooni.protocols.scapy import ScapyTest + +from ooni.lib.txscapy import txsr, txsend + +class scapyArgs(usage.Options): + optParameters = [] + +class ExampleScapyTest(ScapyTest): + """ + An example of writing a scapy Test + """ + implements(IPlugin, ITest) + + shortName = "example_scapy" + description = "An example of a scapy test" + requirements = None + options = scapyArgs + blocking = False + + receive = True + pcapfile = 'example_scapy.pcap' + def initialize(self, reactor=None): + if not self.reactor: + from twisted.internet import reactor + self.reactor = reactor + + self.request = {} + self.response = {} + + def build_packets(self): + """ + Override this method to build scapy packets. + """ + from scapy.all import IP, TCP + return IP()/TCP() + + def load_assets(self): + return {} + +examplescapy = ExampleScapyTest(None, None, None) + diff --git a/ooni/example_plugins/skel.py b/ooni/example_plugins/skel.py new file mode 100644 index 0000000..de95b48 --- /dev/null +++ b/ooni/example_plugins/skel.py @@ -0,0 +1,29 @@ +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from plugoo.tests import ITest, OONITest +from ooni import log + +class SkelArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['other', 'o', None, 'Other arguments']] + +class SkelTest(OONITest): + implements(IPlugin, ITest) + + shortName = "skeleton" + description = "Skeleton plugin" + requirements = None + options = SkelArgs + blocking = False + + def load_assets(self): + if self.local_options: + return {'asset': open(self.local_options['asset'])} + else: + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +skel = SkelTest(None, None, None) diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile index 37abe05..a498a35 100644 --- a/ooni/lib/Makefile +++ b/ooni/lib/Makefile @@ -1,4 +1,4 @@ -all: txtorcon txtraceroute txscapy +all: txtorcon txtraceroute txscapy rfc3339 txtraceroute: echo "Processing dependency txtraceroute..." @@ -18,3 +18,9 @@ txscapy: mv txscapy.git/txscapy.py txscapy.py rm -rf txscapy.git +rfc3339: + echo "Processing RFC3339 dependency" + hg clone https://bitbucket.org/henry/rfc3339 rfc3339 + mv rfc3339/rfc3339.py rfc3339.py + rf -rf rfc3339 + diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 662956a..3eb70f6 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -71,7 +71,6 @@ def runTest(test, options, global_options): reactor=reactor), dict(options), start=resume) - for x in wgen: worker.push(x) diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py index b6b63b6..6ac3c91 100644 --- a/ooni/plugins/blocking.py +++ b/ooni/plugins/blocking.py @@ -27,11 +27,13 @@ class BlockingTest(OONITest): def experiment(self, args): import urllib - req = urllib.urlopen(args['asset']) + url = 'https://torproject.org/' if not 'asset' in args else args['asset'] + req = urllib.urlopen(url) + return {'page': req.readlines()} def load_assets(self): - if self.local_options: + if self.local_options and self.local_options['asset']: return {'asset': Asset(self.local_options['asset'])} else: return {} diff --git a/ooni/plugins/chinatrigger.py b/ooni/plugins/chinatrigger.py index 8a2418a..4f2dc8c 100644 --- a/ooni/plugins/chinatrigger.py +++ b/ooni/plugins/chinatrigger.py @@ -32,7 +32,7 @@ class ChinaTriggerTest(ScapyTest): shortName = "chinatrigger" description = "Triggers the chinese probes into scanning" - requirements = None + requirements = ['root'] options = scapyArgs blocking = False diff --git a/ooni/plugins/examplescapy.py b/ooni/plugins/examplescapy.py deleted file mode 100644 index aa6d81b..0000000 --- a/ooni/plugins/examplescapy.py +++ /dev/null @@ -1,49 +0,0 @@ -import random -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset -from ooni import log -from ooni.protocols.scapy import ScapyTest - -from ooni.lib.txscapy import txsr, txsend - -class scapyArgs(usage.Options): - optParameters = [] - -class ExampleScapyTest(ScapyTest): - """ - An example of writing a scapy Test - """ - implements(IPlugin, ITest) - - shortName = "example_scapy" - description = "An example of a scapy test" - requirements = None - options = scapyArgs - blocking = False - - receive = True - pcapfile = 'example_scapy.pcap' - def initialize(self, reactor=None): - if not self.reactor: - from twisted.internet import reactor - self.reactor = reactor - - self.request = {} - self.response = {} - - def build_packets(self): - """ - Override this method to build scapy packets. - """ - from scapy.all import IP, TCP - return IP()/TCP() - - def load_assets(self): - return {} - -examplescapy = ExampleScapyTest(None, None, None) - diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index 45ab587..ccadff9 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -23,7 +23,7 @@ from ooni.plugoo.tests import ITest, OONITest class HTTPHostArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], - ['controlserver', 'c', None, 'Specify the control server'], + ['controlserver', 'c', 'google.com', 'Specify the control server'], ['resume', 'r', 0, 'Resume at this index'], ['other', 'o', None, 'Other arguments']] def control(self, experiment_result, args): @@ -61,7 +61,7 @@ class HTTPHostTest(OONITest): def check_response(self, response): soup = BeautifulSoup(response) - if soup.head.title.string == "WikiLeaks": + if soup.head.title.string == "Blocked": # Response indicates censorship return True else: @@ -107,13 +107,14 @@ class HTTPHostTest(OONITest): def experiment(self, args): control_server = self.local_options['controlserver'] - censored = self.httplib_test(control_server, args['asset']) + url = 'http://torproject.org/' if not 'asset' in args else args['asset'] + censored = self.httplib_test(control_server, url) return {'control': control_server, - 'host': args['asset'], + 'host': url, 'censored': censored} def load_assets(self): - if self.local_options: + if self.local_options and self.local_options['asset']: return {'asset': Asset(self.local_options['asset'])} else: return {} diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py index 3f92019..d0ede0f 100644 --- a/ooni/plugins/httpt.py +++ b/ooni/plugins/httpt.py @@ -12,7 +12,8 @@ from ooni.protocols import http from ooni import log class httptArgs(usage.Options): - optParameters = [['urls', 'u', None, 'Urls file'], + optParameters = [['urls', 'f', None, 'Urls file'], + ['url', 'u', 'http://torproject.org/', 'Test single site'], ['resume', 'r', 0, 'Resume at this index']] class httptTest(http.HTTPTest): @@ -30,7 +31,7 @@ class httptTest(http.HTTPTest): return {} def load_assets(self): - if self.local_options: + if self.local_options and self.local_options['urls']: return {'url': Asset(self.local_options['urls'])} else: return {} diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py deleted file mode 100644 index de95b48..0000000 --- a/ooni/plugins/skel.py +++ /dev/null @@ -1,29 +0,0 @@ -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from plugoo.tests import ITest, OONITest -from ooni import log - -class SkelArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] - -class SkelTest(OONITest): - implements(IPlugin, ITest) - - shortName = "skeleton" - description = "Skeleton plugin" - requirements = None - options = SkelArgs - blocking = False - - def load_assets(self): - if self.local_options: - return {'asset': open(self.local_options['asset'])} - else: - return {} - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -skel = SkelTest(None, None, None) diff --git a/ooni/plugins/tcpconnect.py b/ooni/plugins/tcpconnect.py index 75f32c1..27df08d 100644 --- a/ooni/plugins/tcpconnect.py +++ b/ooni/plugins/tcpconnect.py @@ -15,7 +15,7 @@ from ooni.plugoo.assets import Asset from ooni import log class tcpconnectArgs(usage.Options): - optParameters = [['blabla', 'a', None, 'Asset file'], + optParameters = [['asset', 'a', None, 'File containing IP:PORT combinations, one per line.'], ['resume', 'r', 0, 'Resume at this index']] class tcpconnectTest(OONITest): @@ -28,7 +28,10 @@ class tcpconnectTest(OONITest): blocking = False def experiment(self, args): - host, port = args['blabla'].split(':') + try: + host, port = args['asset'].split(':') + except: + raise Exception("Error in parsing asset. Wrong format?") class DummyFactory(Factory): def buildProtocol(self, addr): return Protocol() @@ -53,7 +56,7 @@ class tcpconnectTest(OONITest): def load_assets(self): if self.local_options: - return {'blabla': Asset(self.local_options['blabla'])} + return {'asset': Asset(self.local_options['asset'])} else: return {} diff --git a/ooni/plugoo/interface.py b/ooni/plugoo/interface.py index 68cff1a..c3b3855 100644 --- a/ooni/plugoo/interface.py +++ b/ooni/plugoo/interface.py @@ -10,14 +10,37 @@ class ITest(Interface): requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""") - #deferred = Attribute("""This will be fired on test completion""") - #node = Attribute("""This represents the node that will run the test""") options = Attribute("""These are the arguments to be passed to the test for it's execution""") blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""") - def startTest(asset): + def control(experiment_result, args): """ - Launches the Test with the specified arguments on a node. + @param experiment_result: The result returned by the experiment method. + + @param args: the keys of this dict are the names of the assets passed in + from load_assets. The value is one item of the asset. + + Must return a dict containing what should be written to the report. + Anything returned by control ends up inside of the YAMLOONI report. + """ + + def experiment(args): + """ + Perform all the operations that are necessary to running a test. + + @param args: the keys of this dict are the names of the assets passed in + from load_assets. The value is one item of the asset. + + Must return a dict containing the values to be passed to control. + """ + + def load_assets(): + """ + Load the assets that should be passed to the Test. These are the inputs + to the OONI test. + Must return a dict that has as keys the asset names and values the + asset contents. + If the test does not have any assets it should return an empty dict. """ diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index 6140294..63bc3f5 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -44,7 +44,7 @@ class Report: header += "# %s\n\n" % pretty_date self._write_to_report(header) # XXX replace this with something proper - test_details = {'start_time': date.now(), + test_details = {'start_time': str(date.now()), 'asn': 'ASN-1234', 'test_name': self.testname, 'addr': '1234'} diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 608cee3..a3e9150 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -1,16 +1,14 @@ import os -from datetime import datetime import yaml from zope.interface import Interface, Attribute import logging import itertools -import gevent - from twisted.internet import reactor, defer, threads from twisted.python import failure from ooni import log +from ooni import date from ooni.plugoo import assets, work from ooni.plugoo.reports import Report from ooni.plugoo.interface import ITest @@ -38,7 +36,7 @@ class OONITest(object): This method should be overriden by the test writer to provide the logic for loading their assets. """ - return {'asset': None} + return {} def __repr__(self): return "<OONITest %s %s %s>" % (self.options, self.global_options, @@ -46,11 +44,11 @@ class OONITest(object): def finished(self, control): #self.ooninet.report(result) - self.end_time = datetime.now() + self.end_time = date.now() result = self.result - result['start_time'] = self.start_time - result['end_time'] = self.end_time - result['run_time'] = self.end_time - self.start_time + result['start_time'] = str(self.start_time) + result['end_time'] = str(self.end_time) + result['run_time'] = str(self.end_time - self.start_time) result['control'] = control log.msg("FINISHED %s" % result) self.report(result) @@ -68,14 +66,23 @@ class OONITest(object): def control(self, result, args): log.msg("Doing control") - return result + + if self.blocking: + return result + + def end(cb): + return result + d = defer.Deferred() + d.addCallback(end) + return d def experiment(self, args): log.msg("Doing experiment") - return {} + d = defer.Deferred() + return d def startTest(self, args): - self.start_time = datetime.now() + self.start_time = date.now() log.msg("Starting test %s" % self.__class__) return self._do_experiment(args) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 7e9be0c..a3fb168 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -40,12 +40,7 @@ class Worker(object): if isinstance(r, failure.Failure): r.trap() - print r['start_time'] - print r['end_time'] - print r['run_time'] - if self._running == 0 and not self._queued: - print "I am done." reactor.stop() return r @@ -60,62 +55,6 @@ class Worker(object): self._queued.append((workunit, d)) return d -class WorkUnit(object): - """ - XXX This is currently not implemented for KISS sake. - - This is an object responsible for completing WorkUnits it will - return its result in a deferred. - - The execution of a unit of work should be Atomic. - - Reporting to the OONI-net happens on completion of a Unit of Work. - - @Node node: This represents the node associated with the Work Unit - @Asset asset: This is the asset associated with the Work Unit - @Test test: This represents the Test to be with the specified assets - @ivar arguments: These are the extra attributes to be passsed to the Test - """ - - node = None - asset = None - test = None - arguments = None - - def __init__(self, asset, assetNames, test, idx): - self.asset = asset - if not asset: - self.assetGenerator = iter([1]) - else: - self.assetGenerator = iter(asset) - self.Test = test - self.assetNames = assetNames - self.idx = idx - - def __iter__(self): - return self - - def __repr__(self): - return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx) - - def serialize(self): - """ - Serialize this unit of work for RPC activity. - """ - return yaml.dump(self) - - def next(self): - """ - Launches the Unit of Work with the specified assets on the node. - """ - try: - asset = self.assetGenerator.next() - ret = self.Test.set_asset(asset) - return ret - except StopIteration: - raise StopIteration - - class WorkGenerator(object): """ Factory responsible for creating units of work. @@ -129,7 +68,6 @@ class WorkGenerator(object): self.Test = test if self.Test.assets and self.Test.assets.values()[0]: - print self.Test.assets self.assetGenerator = itertools.product(*self.Test.assets.values()) else: self.assetGenerator = None @@ -156,7 +94,7 @@ class WorkGenerator(object): if not self.assetGenerator: self.end = True - return (self.assetNames, self.Test, self.idx) + return ({}, self.Test, self.idx) try: asset = self.assetGenerator.next() @@ -196,3 +134,57 @@ class WorkGenerator(object): self.idx += 1 return WorkUnit(p_asset, self.assetNames, self.Test, self.idx) +class WorkUnit(object): + """ + XXX This is currently not implemented for KISS sake. + + This is an object responsible for completing WorkUnits it will + return its result in a deferred. + + The execution of a unit of work should be Atomic. + + Reporting to the OONI-net happens on completion of a Unit of Work. + + @Node node: This represents the node associated with the Work Unit + @Asset asset: This is the asset associated with the Work Unit + @Test test: This represents the Test to be with the specified assets + @ivar arguments: These are the extra attributes to be passsed to the Test + """ + + node = None + asset = None + test = None + arguments = None + + def __init__(self, asset, assetNames, test, idx): + self.asset = asset + if not asset: + self.assetGenerator = iter([1]) + else: + self.assetGenerator = iter(asset) + self.Test = test + self.assetNames = assetNames + self.idx = idx + + def __iter__(self): + return self + + def __repr__(self): + return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx) + + def serialize(self): + """ + Serialize this unit of work for RPC activity. + """ + return yaml.dump(self) + + def next(self): + """ + Launches the Unit of Work with the specified assets on the node. + """ + try: + asset = self.assetGenerator.next() + ret = self.Test.set_asset(asset) + return ret + except StopIteration: + raise StopIteration diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py index a2081bd..835735b 100644 --- a/ooni/protocols/http.py +++ b/ooni/protocols/http.py @@ -30,7 +30,6 @@ class BodyReceiver(protocol.Protocol): def connectionLost(self, reason): self.finished.callback(self.data) - from twisted.web.http_headers import Headers class HTTPTest(OONITest): """ @@ -77,7 +76,9 @@ class HTTPTest(OONITest): def experiment(self, args): log.msg("Running experiment") - d = self.build_request(args['url']) + url = self.local_options['url'] if 'url' not in args else args['url'] + + d = self.build_request(url) def finished(data): return data

1 0