commit 44859c555f34f6b5e0a65546b476d3ba2e158bad Author: Arturo Filastò art@fuffa.org Date: Mon Nov 26 22:54:42 2012 +0100

Improve the reporting of HTTP Test template. Include the reason of the failure for every request. * Extend config file to support the tor related options * Create ooniprobe.sample * Add ooniprobe.conf to .gitignore --- .gitignore | 1 + ooni/config.py | 15 ++++++++++++++- ooni/templates/httpt.py | 45 ++++++++++++++++++++++++--------------------- ooniprobe.conf | 2 +- ooniprobe.conf.sample | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore index d1015ab..4cb0775 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ pcaps/ report*.yaml docs/build/* data/*.dat +ooniprobe.conf diff --git a/ooni/config.py b/ooni/config.py index edfb196..1c40b32 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -30,12 +30,24 @@ resume_filename = None # There should be a more twisted way of doing this. start_reactor = True

+tor_state = None +tor_control = None + +config_file = None +sample_config_file = None + def get_root_path(): this_directory = os.path.dirname(__file__) root = os.path.join(this_directory, '..') root = os.path.abspath(root) return root

+def createConfigFile(): + """ + XXX implement me + """ + sample_config_file = os.path.join(get_root_path(), 'ooniprobe.conf.sample') + def loadConfigFile(): """ This is a helper function that makes sure that the configuration attributes @@ -45,8 +57,9 @@ def loadConfigFile(): try: f = open(config_file) except IOError: + createConfigFile() raise Exception("Unable to open config file. "\ - "Create a config file called ooniprobe.conf") + "Copy ooniprobe.conf.sample to ooniprobe.conf")

config_file_contents = '\n'.join(f.readlines()) configuration = yaml.safe_load(config_file_contents) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 480d011..b8bd9a7 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -101,7 +101,7 @@ class HTTPTest(NetTestCase): def processInputs(self): pass

- def addToReport(self, request, response=None, response_body=None): + def addToReport(self, request, response=None, response_body=None, failure=None): """ Adds to the report the specified request and response.

@@ -111,6 +111,8 @@ class HTTPTest(NetTestCase): response (instance): An instance of :class:twisted.web.client.Response. Note: headers is our modified True Headers version. + + failure (instance): An instance of :class:twisted.internet.failure.Failure """ log.debug("Adding %s to report" % request) request_response = { @@ -127,6 +129,26 @@ class HTTPTest(NetTestCase): 'body': response_body, 'code': response.code } + if failure: + if isinstance(failure.value, ConnectionRefusedError): + log.err("Connection refused. The backend may be down") + request_response['failure'] = 'connection_refused_error' + + elif isinstance(failure.value, SOCKSError): + log.err("Sock error. The SOCKS proxy may be down") + request_response['failure'] = 'socks_error' + + elif isinstance(failure.value, DNSLookupError): + log.err("DNS lookup failure") + request_response['failure'] = 'dns_lookup_error' + + elif isinstance(failure.value, TCPTimedOutError): + log.err("TCP Timed Out Error") + request_response['failure'] = 'tcp_timed_out_error' + + elif isinstance(failure.value, ResponseNeverReceived): + log.err("Response Never Received") + request_response['failure'] = 'response_never_received' self.report['requests'].append(request_response)

def _processResponseBody(self, response_body, request, response, body_processor): @@ -296,26 +318,7 @@ class HTTPTest(NetTestCase): def errback(failure, request): failure.trap(ConnectionRefusedError, SOCKSError, DNSLookupError, TCPTimedOutError) log.err("Error performing %s" % request) - self.addToReport(request) - if isinstance(failure.value, ConnectionRefusedError): - log.err("Connection refused. The backend may be down") - self.report['failure'] = 'connection_refused_error' - - elif isinstance(failure.value, SOCKSError): - log.err("Sock error. The SOCKS proxy may be down") - self.report['failure'] = 'socks_error' - - elif isinstance(failure.value, DNSLookupError): - log.err("DNS lookup failure") - self.report['failure'] = 'dns_lookup_error' - - elif isinstance(failure.value, TCPTimedOutError): - log.err("TCP Timed Out Error") - self.report['failure'] = 'tcp_timed_out_error' - - elif isinstance(failure.value, ResponseNeverReceived): - log.err("Response Never Received") - self.report['failure'] = 'response_never_received' + self.addToReport(request, failure=failure) return

d = agent.request(request['method'], request['url'], headers, diff --git a/ooniprobe.conf b/ooniprobe.conf index 6777fde..b553a37 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -15,7 +15,7 @@ privacy: # Should we include the ASN of the probe in the report? includecity: false # Should we collect a full packet capture on the client? - includepcap: true + includepcap: false advanced: # XXX change this to point to the directory where you have stored the GeoIP # database file. This should be the directory in which OONI is installed diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample new file mode 100644 index 0000000..a032c0c --- /dev/null +++ b/ooniprobe.conf.sample @@ -0,0 +1,37 @@ +# This is the configuration file for OONIProbe +# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html +# Keep in mind that indentation matters. + +basic: + # Where OONIProbe should be writing it's log file + logfile: ooniprobe.log +privacy: + # Should we include the IP address of the probe in the report? + includeip: false + # Should we include the ASN of the probe in the report? + includeasn: false + # Should we include the ASN of the probe in the report? + includecountry: false + # Should we include the ASN of the probe in the report? + includecity: false + # Should we collect a full packet capture on the client? + includepcap: false +advanced: + # XXX change this to point to the directory where you have stored the GeoIP + # database file. This should be the directory in which OONI is installed + # /path/to/ooni-probe/data/ + geoip_data_dir: /usr/share/GeoIP/ + debug: true + threadpool_size: 10 + tor_binary: '/usr/sbin/tor' + # For auto detection + interface: auto + # Of specify a specific interface + #interface: wlan0 + start_tor: true +tor: + #socks_port: 9050 + #control_port: 9051 + # Specify the absolute path to the Tor bridges to use for testing + bridges: bridges.list +