- tor-commits - lists.torproject.org

[ooni-probe/master] Update HACKING file with updated conventions for code and tests
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 5a68ee6bf83b1ffba2e7baf474f96cb7526f1d88 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 17:23:07 2012 +0100 Update HACKING file with updated conventions for code and tests --- HACKING | 81 ++++++++++++++++++++++++++++++++++++++++++-------------------- 1 files changed, 55 insertions(+), 26 deletions(-) diff --git a/HACKING b/HACKING index 2fdc392..66e6d9e 100644 --- a/HACKING +++ b/HACKING @@ -114,13 +114,67 @@ Code Structure to, where the asset files are located, what should be used for control, etc. +Test related conventions +------------------------ + +These are the conventions for tests that are written in ooniprobe. That is what +goes inside of nettests/. + +Naming +...... + +All methods that are relevant to the test should be all lower case separated by +underscore. + +All variables that are specific to your test should be all lower case separated +by underscore. + +Simplicity +.......... + +Tests written in ooniprobe should be as short as possible and should contain as +little code not related to the measurement as possible. It should be possible +from reading the test to understand what it does without clutter. + +Everything that is not related directly to the test should go inside of the +test template of which the test you are writing is a subclass of. + + Style guide ----------- This is an extract of the most important parts of PEP-8. When in doubt on what code style should be followed first consult this doc, then PEP-8 and if all fails use your best judgement or ask for help. - - Art. + +The most important part to read is the following as it contains the guidelines +of naming of variables, functions and classes, as it does not follow pure +PEP-8. + +Naming convention +................. + +Class names should follow the CapWords convention. +Note: When using abbreviations in CapWords, capitalize all the letters + of the abbreviation. Thus HTTPServerError is better than + HttpServerError. + +Exception names should follow the class names convention as exceptions +should be classes. + +Method names should follow camelCase with the first letter non-capital. + +Class attributes should also follow camelCase with the first letter non-capital. + +Functions should follow camelCase with the first letter non-capital. + +Functions that are inside the local scope of a class or method should be all +lowercase separated by an underscore. + + + + + Indentation ........... @@ -241,28 +295,3 @@ Place docstrings under the def. For a better overview on how to write docstrings consult: PEP-257 -Naming convention -................. - -Avoid using 'l' (lowercase letter el), 'O' (uppercase letter oh) or -I (uppercase letter eye) as single character variable names. - -Module names should have short, all-lowercase names. Underscores can be -used in the module name if it improves readability. Python packages should -also have short, all-lowercase names, although the use of underscores is -discouraged. - -Class names should follow the CapWords convention. -Note: When using abbreviations in CapWords, capitalize all the letters - of the abbreviation. Thus HTTPServerError is better than - HttpServerError. - -Exception names should follow the class names convention as exceptions -should be classes. - -Function names should be all lowercase with words separated by underscores -to improve readability. The same goes for Global Variable names. - -Method names should be all lowercase. Non-public methods should start with -an underscore. The same applies to instance variables. -

1 0

[ooni-probe/master] Fix typo
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 6524c48883ef10721d22c0b2472f2158da1e3e8f Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 17:09:01 2012 +0100 Fix typo --- ooni/config.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index 0aad5cb..ef44ee4 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -36,7 +36,6 @@ def loadConfigFile(): This is a helper function that makes sure that the configuration attributes are singletons. """ - print "LOading config" config_file = os.path.join(get_root_path(), 'ooniprobe.conf') try: f = open(config_file)

1 0

[ooni-probe/master] Make sure that the config file parameters are only instanced once
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 508d045f14f67c1bc182f1f395be253ba7d2a6c2 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 17:07:00 2012 +0100 Make sure that the config file parameters are only instanced once --- ooni/config.py | 61 ++++++++++++++++++++++++++++++++++---------------------- 1 files changed, 37 insertions(+), 24 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index 29255c1..0aad5cb 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -11,6 +11,7 @@ from twisted.internet import reactor, threads from ooni.utils import otime from ooni.utils import Storage + def get_root_path(): this_directory = os.path.dirname(__file__) root = os.path.join(this_directory, '..') @@ -30,30 +31,42 @@ def oreport_filenames(file_name): pcap_filename = base_filename % (test_name, "packets", "pcap") return yamloo_filename, pcap_filename -config_file = os.path.join(get_root_path(), 'ooniprobe.conf') -try: - f = open(config_file) -except IOError: - raise Exception("Unable to open config file. "\ - "Create a config file called ooniprobe.conf") - -config_file_contents = '\n'.join(f.readlines()) -configuration = yaml.safe_load(config_file_contents) - -# Process the basic configuration options -basic = Storage() -for k, v in configuration['basic'].items(): - basic[k] = v - -# Process the privacy configuration options -privacy = Storage() -for k, v in configuration['privacy'].items(): - privacy[k] = v - -# Process the advanced configuration options -advanced = Storage() -for k, v in configuration['advanced'].items(): - advanced[k] = v +def loadConfigFile(): + """ + This is a helper function that makes sure that the configuration attributes + are singletons. + """ + print "LOading config" + config_file = os.path.join(get_root_path(), 'ooniprobe.conf') + try: + f = open(config_file) + except IOError: + raise Exception("Unable to open config file. "\ + "Create a config file called ooniprobe.conf") + + config_file_contents = '\n'.join(f.readlines()) + configuration = yaml.safe_load(config_file_contents) + + # Process the basic configuration options + basic = Storage() + for k, v in configuration['basic'].items(): + basic[k] = v + + # Process the privacy configuration options + privacy = Storage() + for k, v in configuration['privacy'].items(): + privacy[k] = v + + # Process the advanced configuration options + advanced = Storage() + for k, v in configuration['advanced'].items(): + advanced[k] = v + return basic, privacy, advanced + +basic = None +if not basic: + # Here we make sure that we instance the config file attributes only once + basic, privacy, advanced = loadConfigFile() # This is used to keep track of the state of the sniffer sniffer_running = None

1 0

[ooni-probe/master] Add line in oonib to daemonize the script
by art＠torproject.org 21 Nov '12

21 Nov '12

commit e34fc0fd3c0e221d88c45787836d06afa15d0203 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 17:10:07 2012 +0100 Add line in oonib to daemonize the script --- bin/oonib | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/bin/oonib b/bin/oonib index 0808743..f09f790 100755 --- a/bin/oonib +++ b/bin/oonib @@ -23,6 +23,9 @@ from oonib.oonibackend import application sys.argv[1:] = ['-ny', backend_script] +# Uncomment this line to daemonize +#sys.argv[1:] = ['-y', backend_script] + def runApp(config): runner.OBaseRunner(config).run()

1 0

[ooni-probe/master] Add test that detects censorship in HTTP pages based on HTTP body length
by art＠torproject.org 21 Nov '12

21 Nov '12

commit df784323d0da737755def4015840d118e3c8e595 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Nov 20 22:29:00 2012 +0100 Add test that detects censorship in HTTP pages based on HTTP body length --- nettests/core/http_body_length.py | 90 +++++++++++++++++++++++++++++++++++++ 1 files changed, 90 insertions(+), 0 deletions(-) diff --git a/nettests/core/http_body_length.py b/nettests/core/http_body_length.py new file mode 100644 index 0000000..7c5b0d2 --- /dev/null +++ b/nettests/core/http_body_length.py @@ -0,0 +1,90 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from twisted.internet import defer +from twisted.python import usage +from ooni.templates import httpt + +class UsageOptions(usage.Options): + optParameters = [ + ['url', 'u', None, 'Specify a single URL to test.'], + ['factor', 'f', 0.8, 'What factor should be used for triggering censorship (0.8 == 80%)'] + ] + +class HTTPBodyLength(httpt.HTTPTest): + """ + Performs a two GET requests to the set of sites to be tested for + censorship, one over a known good control channel (Tor), the other over the + test network. + We then look at the response body lengths and see if the control response + differs from the experiment response by a certain factor. + """ + name = "HTTP Body length test" + author = "Arturo Filastò" + version = "0.1" + + usageOptions = UsageOptions + + inputFile = ['file', 'f', None, + 'List of URLS to perform GET and POST requests to'] + + # These values are used for determining censorship based on response body + # lengths + control_body_length = None + experiment_body_length = None + + def setUp(self): + """ + Check for inputs. + """ + if self.input: + self.url = self.input + elif self.localOptions['url']: + self.url = self.localOptions['url'] + else: + raise Exception("No input specified") + + self.factor = self.localOptions['factor'] + + def compare_body_lengths(self): + body_length_a = self.control_body_length + body_length_b = self.experiment_body_length + + rel = float(body_length_a)/float(body_length_b) + if rel > 1: + rel = 1/rel + + self.report['body_proportion'] = rel + self.report['factor'] = self.factor + if rel < self.factor: + self.report['censorship'] = True + else: + self.report['censorship'] = False + + def test_get(self): + def errback(failure): + log.err("There was an error while testing %s" % self.url) + log.exception(failure) + + def control_body(result): + self.control_body_length = len(result) + if self.experiment_body_length: + self.compare_body_lengths() + + def experiment_body(result): + self.experiment_body_length = len(result) + if self.control_body_length: + self.compare_body_lengths() + + dl = [] + experiment_request = self.doRequest(self.url, method="GET", + body_processor=experiment_body) + control_request = self.doRequest(self.url, method="GET", + use_tor=True, body_processor=control_body) + dl.append(experiment_request) + dl.append(control_request) + d = defer.DeferredList(dl) + return d +

1 0

[ooni-probe/master] Better error handling in reporter
by art＠torproject.org 21 Nov '12

21 Nov '12

commit b243272496665e9d55929bd9c2f5015e0565fc56 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 11:25:51 2012 +0100 Better error handling in reporter * Notify the user when the connection has failed because the backend was not reachable --- nettests/core/http_host.py | 2 +- ooni/reporter.py | 12 ++++++++---- ooni/runner.py | 19 ++++++++++++++++--- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/nettests/core/http_host.py b/nettests/core/http_host.py index 3912dfc..3ebfd04 100644 --- a/nettests/core/http_host.py +++ b/nettests/core/http_host.py @@ -35,7 +35,7 @@ class HTTPHost(httpt.HTTPTest): usageOptions = UsageOptions - inputFile = ['file', 'f', None, + inputFile = ['file', 'f', None, 'List of hostnames to test for censorship'] requiredOptions = ['backend'] diff --git a/ooni/reporter.py b/ooni/reporter.py index 5006a21..0110b02 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -18,6 +18,7 @@ import traceback from twisted.python.util import untilConcludes from twisted.trial import reporter from twisted.internet import defer, reactor +from twisted.internet.error import ConnectionRefusedError from ooni.templates.httpt import BodyReceiver, StringProducer from ooni.utils import otime, log, geodata @@ -204,18 +205,18 @@ class OONIBReporter(OReporter): request = {'report_id': self.report_id, 'content': content} - log.debug("Updating report with id %s" % self.report_id) + log.debug("Updating report with id %s (%s)" % (self.report_id, url)) request_json = json.dumps(request) log.debug("Sending %s" % request_json) bodyProducer = StringProducer(json.dumps(request)) - log.debug("Creating report via url %s" % url) try: response = yield self.agent.request("PUT", url, bodyProducer=bodyProducer) except: # XXX we must trap this in the runner and make sure to report the data later. + log.err("Error in writing report entry") raise OONIBReportUpdateFailed #parsed_response = json.loads(backend_response) @@ -254,12 +255,15 @@ class OONIBReporter(OReporter): log.debug("Sending %s" % request_json) bodyProducer = StringProducer(json.dumps(request)) - log.debug("Creating report via url %s" % url) try: response = yield self.agent.request("POST", url, bodyProducer=bodyProducer) - except: + except ConnectionRefusedError: + log.err("Connection to reporting backend failed (ConnectionRefusedError)") + raise OONIBReportCreationFailed + except Exception, e: + log.exception(e) raise OONIBReportCreationFailed # This is a little trix to allow us to unspool the response. We create diff --git a/ooni/runner.py b/ooni/runner.py index 536ecfe..8edf857 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -207,18 +207,31 @@ def runTestCases(test_cases, options, log.msg("options[0] = %s" % first) test_inputs = [None] - reportFile = open(yamloo_filename, 'w+') + log.debug("Creating %s" % yamloo_filename) if cmd_line_options['collector']: + log.debug("Using remote collector %s" % cmd_line_options['collector']) oreporter = reporter.OONIBReporter(cmd_line_options['collector']) else: + reportFile = open(yamloo_filename, 'w+') + log.debug("Reporting to file %s" % reportFile) oreporter = reporter.YAMLReporter(reportFile) - input_unit_factory = InputUnitFactory(test_inputs) + try: + input_unit_factory = InputUnitFactory(test_inputs) + except Exception, e: + log.exception(e) log.debug("Creating report") - yield oreporter.createReport(options) + try: + yield oreporter.createReport(options) + except reporter.OONIBReportCreationFailed: + log.err("Error in creating new report") + reactor.stop() + raise + except Exception, e: + log.exception(e) # This deferred list is a deferred list of deferred lists # it is used to store all the deferreds of the tests that

1 0

[ooni-probe/master] Make the logger start later to avoid having stdout prefixed with INFO:twisted
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 32d541672b58eb6eb412148aeceef14656762560 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 11:24:26 2012 +0100 Make the logger start later to avoid having stdout prefixed with INFO:twisted * This makes the help messages of tests more readable and less clutterred --- ooni/oonicli.py | 19 +++++++++---------- 1 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 9473ad8..8c5c0e9 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -95,10 +95,7 @@ def run(): if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) - log.start(cmd_line_options['logfile']) - test_file_name = os.path.basename(cmd_line_options['test']) - log.debug("Running script %s" % test_file_name) yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name) @@ -107,12 +104,12 @@ def run(): pcap_filename = yamloo_filename+".pcap" if os.path.exists(yamloo_filename): - log.msg("Report already exists with filename %s" % yamloo_filename) - log.msg("Renaming it to %s" % yamloo_filename+'.old') + print "Report already exists with filename %s" % yamloo_filename + print "Renaming it to %s" % yamloo_filename+'.old' os.rename(yamloo_filename, yamloo_filename+'.old') if os.path.exists(pcap_filename): - log.msg("Report already exists with filename %s" % pcap_filename) - log.msg("Renaming it to %s" % pcap_filename+'.old') + print "Report PCAP already exists with filename %s" % pcap_filename + print "Renaming it to %s" % pcap_filename+'.old' os.rename(pcap_filename, pcap_filename+'.old') classes = runner.findTestClassesFromConfig(cmd_line_options) @@ -121,12 +118,14 @@ def run(): try: checkForRoot() except NotRootError: - log.err("includepcap options requires root priviledges to run") - log.err("you should run ooniprobe as root or disable the options in ooniprobe.conf") + print "[!] Includepcap options requires root priviledges to run" + print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) - log.debug("Starting sniffer") + print "Starting sniffer" sniffer_d = net.capturePackets(pcap_filename) + log.start(cmd_line_options['logfile']) + tests_d = runner.runTestCases(test_cases, options, cmd_line_options, yamloo_filename) tests_d.addBoth(testsEnded)

1 0

[ooni-probe/master] Make the reporting directory of the collector configurable
by art＠torproject.org 21 Nov '12

21 Nov '12

commit d9ff0d3b1c8412a1254a91128ec6f423ce41361c Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 11:26:32 2012 +0100 Make the reporting directory of the collector configurable --- oonib/config.py | 1 + oonib/report/api.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/oonib/config.py b/oonib/config.py index 1a70b85..e07dc98 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -10,6 +10,7 @@ def get_root_path(): # XXX convert this to something that is a proper config file main = Storage() main.collector_port = 8888 +main.report_dir = os.path.join(get_root_path(), 'oonib', 'reports') # XXX make this be the server name that is of main.database_uri = "sqlite:"+get_root_path()+"oonib_test_db.db" diff --git a/oonib/report/api.py b/oonib/report/api.py index bd409b0..1a81e68 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -10,13 +10,14 @@ import random import string import json import re -from ooni.utils import randomStr +import os from twisted.internet import reactor, defer from cyclone import web -from oonib import models +from ooni.utils import randomStr +from oonib import models, config backend_version = '0.0.1' @@ -75,7 +76,7 @@ def parseUpdateReportRequest(request): report_id = parsed_request['report_id'] except KeyError: raise MissingField('report_id') - + if not re.match(report_id_regexp, report_id): raise InvalidRequestField('report_id') @@ -131,7 +132,7 @@ class NewReportHandlerFile(web.RequestHandler): # report_data['test_name'], # report_data['test_version'])) - report_filename = report_id + report_filename = os.path.join(config.main.report_dir, report_id) report_filename += '.yamloo' response = {'backend_version': backend_version, @@ -155,7 +156,7 @@ class NewReportHandlerFile(web.RequestHandler): report_id = parsed_request['report_id'] print "Got this request %s" % parsed_request - report_filename = report_id + report_filename = os.path.join(config.main.report_dir, report_id) report_filename += '.yamloo' try: with open(report_filename, 'a+') as f:

1 0

[ooni-probe/master] Rename sockport and sockhost parameters in txagentwithsocks to socksport and sockshost
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 86f914d18a4e6d76365e3ec35d50f35267865279 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 11:23:37 2012 +0100 Rename sockport and sockhost parameters in txagentwithsocks to socksport and sockshost --- ooni/lib/txagentwithsocks.py | 21 ++++++++++++--------- ooni/reporter.py | 6 ++++-- ooni/templates/httpt.py | 8 ++++---- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/ooni/lib/txagentwithsocks.py b/ooni/lib/txagentwithsocks.py index a1f7f48..fc1c17c 100644 --- a/ooni/lib/txagentwithsocks.py +++ b/ooni/lib/txagentwithsocks.py @@ -98,12 +98,12 @@ class SOCKSv5ClientFactory(_WrappingFactory): class SOCKS5ClientEndpoint(object): implements(interfaces.IStreamClientEndpoint) - def __init__(self, reactor, sockhost, sockport, + def __init__(self, reactor, sockshost, socksport, host, port, timeout=30, bindAddress=None): self._reactor = reactor - self._sockhost = sockhost - self._sockport = sockport + self._sockshost = sockshost + self._socksport = socksport self._host = host self._port = port self._timeout = timeout @@ -113,7 +113,7 @@ class SOCKS5ClientEndpoint(object): try: wf = SOCKSv5ClientFactory(protocolFactory, self._host, self._port) self._reactor.connectTCP( - self._sockhost, self._sockport, wf, + self._sockshost, self._socksport, wf, timeout=self._timeout, bindAddress=self._bindAddress) return wf._onConnection except: @@ -204,7 +204,7 @@ class Agent(client.Agent): def __init__(self, reactor, contextFactory=client.WebClientContextFactory(), connectTimeout=None, bindAddress=None, - pool=None, sockhost=None, sockport=None): + pool=None, sockshost=None, socksport=None): if pool is None: pool = HTTPConnectionPool(reactor, False) self._reactor = reactor @@ -212,8 +212,8 @@ class Agent(client.Agent): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress - self._sockhost = sockhost - self._sockport = sockport + self._sockshost = sockshost + self._socksport = socksport def _getEndpoint(self, scheme, host, port): kwargs = {} @@ -223,8 +223,11 @@ class Agent(client.Agent): if scheme == 'http': return TCP4ClientEndpoint(self._reactor, host, port, **kwargs) elif scheme == 'shttp': - return SOCKS5ClientEndpoint(self._reactor, self._sockhost, - self._sockport, host, port, **kwargs) + return SOCKS5ClientEndpoint(self._reactor, self._sockshost, + self._socksport, host, port, **kwargs) + elif scheme == 'httpo': + return SOCKS5ClientEndpoint(self._reactor, self._sockshost, + self._socksport, host, port, **kwargs) elif scheme == 'https': return SSL4ClientEndpoint(self._reactor, host, port, self._wrapContextFactory(host, port), diff --git a/ooni/reporter.py b/ooni/reporter.py index 05ea94a..5006a21 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -185,9 +185,11 @@ class OONIBTestDetailsLookupFailed(Exception): class OONIBReporter(OReporter): def __init__(self, backend_url): - from twisted.web.client import Agent + from ooni.lib.txagentwithsocks import Agent from twisted.internet import reactor - self.agent = Agent(reactor) + + self.agent = Agent(reactor, sockshost="127.0.0.1", + socksport=config.advanced.socksport) self.backend_url = backend_url @defer.inlineCallbacks diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 7caab39..c62ea4d 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -59,16 +59,16 @@ class HTTPTest(NetTestCase): log.err("Warning! pyOpenSSL is not installed. https websites will" "not work") - self.control_agent = Agent(reactor, sockhost="127.0.0.1", - sockport=config.advanced.tor_socksport) + self.control_agent = Agent(reactor, sockshost="127.0.0.1", + socksport=config.advanced.tor_socksport) sockshost, socksport = (None, None) if self.localOptions['socksproxy']: sockshost, socksport = self.localOptions['socksproxy'].split(':') socksport = int(socksport) - self.agent = Agent(reactor, sockhost=sockshost, - sockport=socksport) + self.agent = Agent(reactor, sockshost=sockshost, + socksport=socksport) if self.followRedirects: try:

1 0

[ooni-probe/master] Implement first iteration of Tor Hidden Service support for ooniprobe and oonib
by art＠torproject.org 21 Nov '12

21 Nov '12

commit d0d9662b4cafe11979e8173e6cc03c548f99ecf0 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 14:49:29 2012 +0100 Implement first iteration of Tor Hidden Service support for ooniprobe and oonib * Also with better error handling for reports --- ooni/reporter.py | 16 +++++++++++--- oonib/oonibackend.py | 17 ++++----------- oonib/runner.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 0110b02..0635d2f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -188,9 +188,12 @@ class OONIBReporter(OReporter): def __init__(self, backend_url): from ooni.lib.txagentwithsocks import Agent from twisted.internet import reactor + try: + self.agent = Agent(reactor, sockshost="127.0.0.1", + socksport=int(config.advanced.tor_socksport)) + except Exception, e: + log.exception(e) - self.agent = Agent(reactor, sockshost="127.0.0.1", - socksport=config.advanced.socksport) self.backend_url = backend_url @defer.inlineCallbacks @@ -257,7 +260,7 @@ class OONIBReporter(OReporter): bodyProducer = StringProducer(json.dumps(request)) try: - response = yield self.agent.request("POST", url, + response = yield self.agent.request("POST", url, bodyProducer=bodyProducer) except ConnectionRefusedError: log.err("Connection to reporting backend failed (ConnectionRefusedError)") @@ -273,7 +276,12 @@ class OONIBReporter(OReporter): backend_response = yield response_body - parsed_response = json.loads(backend_response) + try: + parsed_response = json.loads(backend_response) + except Exception, e: + log.exception(e) + raise OONIBReportCreationFailed + self.report_id = parsed_response['report_id'] self.backend_version = parsed_response['backend_version'] log.debug("Created report with id %s" % parsed_response['report_id']) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index dd25c8c..c712bc4 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -13,19 +13,17 @@ from twisted.application import internet, service from twisted.application.service import Application from twisted.names import dns -from ooni.utils import log +from cyclone import web -from oonib.report.api import reportingBackend - -from oonib import config +import txtorcon from oonib.testhelpers import dns_helpers, ssl_helpers from oonib.testhelpers import http_helpers, tcp_helpers -#from oonib.testhelpers.daphn3 import Daphn3Server -from oonib import db_threadpool +from ooni.utils import log -from cyclone import web +from oonib import db_threadpool +from oonib import config application = service.Application('oonibackend') serviceCollection = service.IServiceCollection(application) @@ -59,11 +57,6 @@ if config.helpers.daphn3.port: tcp_helpers.Daphn3Server()) daphn3_helper.setServiceParent(serviceCollection) -if config.main.collector_port: - print "Starting Collector on %s" % config.main.collector_port - collector = internet.TCPServer(int(config.main.collector_port), - reportingBackend) - collector.setServiceParent(serviceCollection) if config.helpers.tcp_echo.port: print "Starting TCP echo helper on %s" % config.helpers.tcp_echo.port diff --git a/oonib/runner.py b/oonib/runner.py index febef46..2538710 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -4,11 +4,45 @@ We are just extending the """ +from twisted.internet import reactor from twisted.application import service, internet, app from twisted.python.runtime import platformType +import txtorcon + +from oonib.report.api import reportingBackend + +from oonib import config from ooni.utils import log +def txSetupFailed(failure): + log.err("Setup failed") + log.exception(failure) + +def setupCollector(tor_process_protocol): + def setup_complete(port): + print "Exposed collector Tor hidden service on httpo://%s" % port.onion_uri + + torconfig = txtorcon.TorConfig(tor_process_protocol.tor_protocol) + public_port = 80 + hs_endpoint = txtorcon.TCPHiddenServiceEndpoint(reactor, torconfig, + public_port) + hidden_service = hs_endpoint.listen(reportingBackend) + hidden_service.addCallback(setup_complete) + hidden_service.addErrback(txSetupFailed) + +def startTor(): + def updates(prog, tag, summary): + print "%d%%: %s" % (prog, summary) + + torconfig = txtorcon.TorConfig() + torconfig.SocksPort = 9055 + torconfig.save() + d = txtorcon.launch_tor(torconfig, reactor, + progress_updates=updates) + d.addCallback(setupCollector) + d.addErrback(txSetupFailed) + class OBaseRunner(): pass @@ -17,9 +51,24 @@ if platformType == "win32": WindowsApplicationRunner OBaseRunner = WindowsApplicationRunner + else: from twisted.scripts._twistd_unix import ServerOptions, \ UnixApplicationRunner - OBaseRunner = UnixApplicationRunner + class OBaseRunner(UnixApplicationRunner): + def postApplication(self): + """ + To be called after the application is created: start the + application and run the reactor. After the reactor stops, + clean up PID files and such. + """ + self.startApplication(self.application) + # This is our addition. The rest is taken from + # twisted/scripts/_twistd_unix.py 12.2.0 + startTor() + self.startReactor(None, self.oldstdout, self.oldstderr) + self.removePID(self.config['pidfile']) OBaseRunner.loggerFactory = log.LoggerFactory + +

1 0