June 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge branch 'feature/task_manager' into feature/ui
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 81c53bbc6107d955731eabe6d41e24c586d07122 Merge: 3b88935 056a24b Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 29 18:15:47 2013 +0200 Merge branch 'feature/task_manager' into feature/ui * feature/task_manager: Make how we keep track of successful reports more robust. ooni/reporter.py | 12 ++++++------ ooni/tasks.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --cc ooni/reporter.py index 1095cb1,732d516..5d8f92b --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -32,9 -32,9 +32,9 @@@ from ooni import otim from ooni.utils import geodata, pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents -from ooni import config +from ooni.settings import config - from ooni.tasks import ReportEntry, TaskTimedOut + from ooni.tasks import ReportEntry, TaskTimedOut, ReportTracker class ReporterException(Exception): pass diff --cc ooni/tasks.py index f686a9c,428fdbf..35f1236 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@@ -129,7 -129,28 +129,26 @@@ class Measurement(TaskWithTimeout) d = self.netTestMethod() return d + class ReportTracker(object): + def __init__(self, reporters): + self.report_completed = 0 + self.reporters = reporters + + def finished(self): + """ + Returns true if all the tasks are done. False if not. + """ + if self.report_completed == len(self.reporters): + return True + return False + + def completed(self): + """ + Called when a new report is completed. + """ + self.report_completed += 1 + class ReportEntry(TaskWithTimeout): - timeout = config.advanced.reporting_timeout - def __init__(self, reporter, measurement): self.reporter = reporter self.measurement = measurement

1 0

[ooni-probe/master] Implement in-browser input file selection views
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 13a65fd6adb798f6c2bdcc4584a5cff52e694d7c Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 29 18:54:01 2013 +0200 Implement in-browser input file selection views This will allow the user to copy and paste the input file into a textarea. --- data/ui/app/scripts/controllers.js | 28 ++++++++++++++++++++++++++++ data/ui/app/views/test.html | 12 +++++++++--- ooni/api/spec.py | 33 +++++++++++++++++++++++++++++++-- ooni/nettest.py | 5 ++++- 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/data/ui/app/scripts/controllers.js b/data/ui/app/scripts/controllers.js index 5ba24e1..c2a9c0d 100644 --- a/data/ui/app/scripts/controllers.js +++ b/data/ui/app/scripts/controllers.js @@ -49,7 +49,17 @@ ooniprobe.controller('TestCtrl', ['$scope', '$routeParams', 'testStatus', 'Input ooniprobe.controller('TestBoxCtrl', ['$scope', 'startTest', function($scope, startTest) { + function hasAttributes(obj) { + var count = 0; + for (var i in obj) + count +=1; + if ( count == 0 ) { + return false; + } + return true; + } + $scope.manualFileInput = {}; $scope.startTest = function() { var options = {}; @@ -58,6 +68,13 @@ ooniprobe.controller('TestBoxCtrl', ['$scope', 'startTest', options[key] = option.value; }); + if (hasAttributes($scope.manualFileInput)) { + options['manual_input'] = {}; + angular.forEach($scope.manualFileInput, function(value, key) { + options['manual_input'][key] = value; + }); + } + startTest($scope.testDetails.id, options).success(function(){ $scope.updateTestStatus(); }); @@ -65,4 +82,15 @@ ooniprobe.controller('TestBoxCtrl', ['$scope', 'startTest', }]); +ooniprobe.controller('FileInput', ['$scope', + function($scope) { + + $scope.manualShow = false; + $scope.toggleManualInput = function() { + if ($scope.manualShow) + $scope.manualShow = false; + else + $scope.manualShow = true; + } +}]); diff --git a/data/ui/app/views/test.html b/data/ui/app/views/test.html index f92461a..fdd5931 100644 --- a/data/ui/app/views/test.html +++ b/data/ui/app/views/test.html @@ -8,11 +8,17 @@ <div ng-repeat="(name, options) in testDetails.arguments"> <div ng-switch on="options.type"> - <div ng-switch-when="file"> - <label>{{name}}</label> - <select ng-model="testDetails.arguments[name].value"> + <div ng-switch-when="file" ng-controller="FileInput"> + <label>{{name}} + <button class="btn btn-small" ng-click="toggleManualInput()">Toggle manual input</button> + </label> + <select ng-model="testDetails.arguments[name].value" ng-hide="manualShow"> <option ng-repeat="input in inputs" value="input.filename">{{input.filename}}</option> </select> + <br/> + <div class="manualFileInput" ng-show="manualShow"> + <textarea ng-model="manualFileInput[name]"></textarea> + </div> </div> <div ng-switch-default> diff --git a/ooni/api/spec.py b/ooni/api/spec.py index 071beaf..fe18ccc 100644 --- a/ooni/api/spec.py +++ b/ooni/api/spec.py @@ -3,6 +3,7 @@ import re import copy import json import types +import tempfile from twisted.python import usage from cyclone import web, escape @@ -122,6 +123,20 @@ def get_reporters(net_test_loader): reporters.append(oonib_reporter) return reporters +def write_temporary_input(content): + """ + Creates a temporary file for the given content. + + Returns: + the path to the temporary file. + """ + fd, path = tempfile.mkstemp() + with open(path, 'w') as f: + f.write(content) + f.close() + print "This is the path %s" % path + return fd, path + class StartTest(ORequestHandler): def post(self, test_name): """ @@ -129,11 +144,25 @@ class StartTest(ORequestHandler): """ test_file = oonidApplication.director.netTests[test_name]['path'] test_options = json.loads(self.request.body) + tmp_files = [] + if (test_options['manual_input']): + for option, content in test_options['manual_input'].items(): + fd, path = write_temporary_input(content) + test_options[option] = path + tmp_files.append((fd, path)) + test_options.pop('manual_input') + net_test_loader = get_net_test_loader(test_options, test_file) try: net_test_loader.checkOptions() - oonidApplication.director.startNetTest(net_test_loader, - get_reporters(net_test_loader)) + d = oonidApplication.director.startNetTest(net_test_loader, + get_reporters(net_test_loader)) + @d.addBoth + def cleanup(result): + for fd, path in tmp_files: + os.close(fd) + os.remove(path) + except MissingRequiredOption, option_name: self.write({'error': 'Missing required option: "%s"' % option_name}) diff --git a/ooni/nettest.py b/ooni/nettest.py index 66639f8..601fa24 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -132,8 +132,11 @@ def getArguments(test_class): for opt_parameter in test_class.usageOptions.optParameters: option_name = opt_parameter[0] + opt_type="text" + if opt_parameter[3].lower().startswith("file"): + opt_type="file" arguments[option_name] = getOption(opt_parameter, - test_class.requiredOptions) + test_class.requiredOptions, type=opt_type) return arguments

1 0

[ooni-probe/master] Merge branch 'master' into feature/ui
by art＠torproject.org 19 Jun '13

19 Jun '13

commit c8ee22a8afc3fd7f3525d1f1d4e717f3aadd2713 Merge: 673236c 5782f32 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun May 5 17:52:44 2013 +0200 Merge branch 'master' into feature/ui * master: (69 commits) Implement Vagrant file for running ooniprobe on any platform Have some slightly saner defaults for the config file Fix markup of README file Fix markup of code Swallow the report creation failure so that it does not propagate Fix tls_handshake input processing Fix import conflict Set self.input to self.host if no inputFile, and update several log messages. Reformat docstrings for several HandshakeTest functions to Sphinx format. Rename HandshakeTest.test_tlsv1_handshake() to HandshakeTest.test_handshake(). Cleanup docstrings for getX509Name() and getPublicKey() functions. Add Sphinx docstring for getPeerCert() function. Move to call IPAddress explicitly from ipaddr.IPAdress. Add self.host to the inputProcessor(). If no ciphersuite list is provided, explain that we're using firefox's. Fix bug where conflicting SSL context choices was not explained to the user. Fix #8440, set the self.host attribute when given the --host option. Create a host attribute for storing the --host option, and set self.ciphers. Make 'context' be a class-level attribute and give it a Sphinx :attr: docstring. Bump the version number on HandshakeTest, and rename the usageOptions attribute. ... Conflicts: data/ooniprobe.conf.sample ooni/director.py .gitignore | 2 + README.md | 161 +++++-- Vagrantfile | 153 ++++++ data/ooniprobe.conf.sample | 4 +- docs/fabfile.py | 22 + inputs/Makefile | 3 +- nettests/experimental/tls_handshake.py | 794 ++++++++++++++++++++++++++++++++ ooni/reporter.py | 3 +- ooni/utils/txscapy.py | 1 + setup.py | 4 +- 10 files changed, 1089 insertions(+), 58 deletions(-) diff --cc data/ooniprobe.conf.sample index 2b79d3e,0000000..a12b526 mode 100644,000000..100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@@ -1,55 -1,0 +1,55 @@@ +# This is the configuration file for OONIProbe +# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html +# Keep in mind that indentation matters. + +basic: + # Where OONIProbe should be writing it's log file + logfile: /var/log/ooniprobe.log +privacy: + # Should we include the IP address of the probe in the report? + includeip: false + # Should we include the ASN of the probe in the report? + includeasn: false + # Should we include the country as reported by GeoIP in the report? + includecountry: false + # Should we include the city as reported by GeoIP in the report? + includecity: false + # Should we collect a full packet capture on the client? + includepcap: false +reports: + # This is a packet capture file (.pcap) to load as a test: + pcap: null + collector: 'httpo://nkvphnp3p6agi5qq.onion' +advanced: + geoip_data_dir: /usr/share/ooni/ + debug: true + # enable if auto detection fails + #tor_binary: /usr/sbin/tor + # For auto detection + interface: auto + # Of specify a specific interface + #interface: wlan0 + # If you do not specify start_tor, you will have to have Tor running and + # explicitly set the control port and SOCKS port - start_tor: true ++ start_tor: false + # After how many seconds we should give up on a particular measurement + measurement_timeout: 30 + # After how many retries we should give up on a measurement + measurement_retries: 2 + # How many measurments to perform concurrently + measurement_concurrency: 100 + # After how may seconds we should give up reporting + reporting_timeout: 30 + # After how many retries to give up on reporting + reporting_retries: 3 + # How many reports to perform concurrently + reporting_concurrency: 20 +tor: - #socks_port: 9050 ++ socks_port: 9050 + #control_port: 9051 + # Specify the absolute path to the Tor bridges to use for testing + #bridges: bridges.list + # Specify path of the tor datadirectory. + # This should be set to something to avoid having Tor download each time + # the descriptors and consensus data. + #data_dir: ~/.tor/ diff --cc setup.py index 454984d,e336e08..1ae39d4 --- a/setup.py +++ b/setup.py @@@ -13,19 -11,9 +13,17 @@@ install_requires = ] dependency_links = [ - 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe', - 'https://github.com/hellais/pypcap/archive/v1.1.1.tar.gz#egg=pypcap-1.1.1' + 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe' ] +files = [] +for root, dirs, file_names in os.walk('data/'): + for file_name in file_names: + if not file_name.endswith('.pyc'): + files.append(os.path.join(root, file_name)) + +data_files = [('/usr/share/ooni/', files)] + with open('requirements.txt') as f: for line in f: if line.startswith("#") or line.startswith('http'):

1 0

[ooni-probe/master] Add cyclone as a dependency
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 28388e3522c3b82200f7afd2b748e2ac60525ead Author: Arturo Filastò <art(a)fuffa.org> Date: Tue May 7 18:59:27 2013 +0200 Add cyclone as a dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index bdf4cd0..4f951de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ zope.component>=4.0.0 zope.event>=4.0.0 zope.interface>=4.0.1 Pyrex +cyclone>=1.1 # # This is a Tor Project mirror with valid SSL/TLS certs that is stable and fast #

1 0

[ooni-probe/master] Implement XSRF protection in the HTTP API
by art＠torproject.org 19 Jun '13

19 Jun '13

commit ccf125bce7cd3094bfd4bb8b1ce6b8d2d6890e22 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue May 7 18:59:10 2013 +0200 Implement XSRF protection in the HTTP API --- ooni/api/spec.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ooni/api/spec.py b/ooni/api/spec.py index fe18ccc..f707253 100644 --- a/ooni/api/spec.py +++ b/ooni/api/spec.py @@ -4,6 +4,7 @@ import copy import json import types import tempfile +import functools from twisted.python import usage from cyclone import web, escape @@ -19,8 +20,18 @@ class InvalidInputFilename(Exception): class FilenameExists(Exception): pass +def check_xsrf(method): + @functools.wraps(method) + def wrapper(self, *args, **kw): + xsrf_header = self.request.headers.get("X-XSRF-TOKEN") + if self.xsrf_token != xsrf_header: + raise web.HTTPError(403, "Invalid XSRF token.") + return method(self, *args, **kw) + return wrapper + class ORequestHandler(web.RequestHandler): serialize_lists = True + xsrf_cookie_name = "XSRF-TOKEN" def write(self, chunk): """ @@ -33,6 +44,7 @@ class ORequestHandler(web.RequestHandler): web.RequestHandler.write(self, chunk) class Status(ORequestHandler): + @check_xsrf def get(self): result = {'active_tests': oonidApplication.director.activeNetTests} self.write(result) @@ -47,6 +59,8 @@ class Inputs(ORequestHandler): """ This handler is responsible for listing and adding new inputs. """ + + @check_xsrf def get(self): """ Obtain the list of currently installed inputs. Inputs are stored inside @@ -55,6 +69,7 @@ class Inputs(ORequestHandler): input_list = list_inputs() self.write(input_list) + @check_xsrf def post(self): """ Add a new input to the currently installed inputs. @@ -76,6 +91,8 @@ class Inputs(ORequestHandler): fp.write(body) class ListTests(ORequestHandler): + + @check_xsrf def get(self): test_list = copy.deepcopy(oonidApplication.director.netTests) for test_id in test_list.keys(): @@ -138,6 +155,8 @@ def write_temporary_input(content): return fd, path class StartTest(ORequestHandler): + + @check_xsrf def post(self, test_name): """ Starts a test with the specified options. @@ -174,6 +193,8 @@ class StartTest(ORequestHandler): 'Insufficient priviledges'}) class StopTest(ORequestHandler): + + @check_xsrf def delete(self, test_name): pass @@ -199,6 +220,8 @@ def get_test_results(test_id): return test_results class TestStatus(ORequestHandler): + + @check_xsrf def get(self, test_id): """ Returns the requested test_id details and the stored results for such

1 0

[ooni-probe/master] Bump to version 0.1.0
by art＠torproject.org 19 Jun '13

19 Jun '13

commit b73fa7c06a9fbd59ba1a792200e767e1b03ff669 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed May 8 16:44:02 2013 +0200 Bump to version 0.1.0 --- ooni/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 1810a7f..9663b08 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -7,7 +7,7 @@ from . import templates from . import utils __author__ = "Arturo Filastò" -__version__ = "0.0.12" +__version__ = "0.1.0" __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', diff --git a/setup.py b/setup.py index 1ae39d4..6ebca7b 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ with open('requirements.txt') as f: setup( name="ooni-probe", - version="0.0.12", + version="0.1.0", author="Arturo Filastò", author_email = "art(a)torproject.org", url="https://ooni.torproject.org/",

1 0

[ooni-probe/master] Include jquery as dep
by art＠torproject.org 19 Jun '13

19 Jun '13

commit ccc53b5cd5d7cf1e96b9595b0e5d380ca6dd69e9 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed May 8 19:10:58 2013 +0200 Include jquery as dep Fix bug in ng-upload --- data/ui/app/index.html | 1 + data/ui/app/libs/ng-upload/ng-upload.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data/ui/app/index.html b/data/ui/app/index.html index c304066..5f3e376 100644 --- a/data/ui/app/index.html +++ b/data/ui/app/index.html @@ -28,6 +28,7 @@ <script src="libs/angular/angular.js"></script> <script src="libs/angular-resource/angular-resource.js"></script> + <script src="libs/jquery/jquery.js"></script> <script src="libs/ng-upload/ng-upload.js"></script> <script src="scripts/app.js"></script> <script src="scripts/services.js"></script> diff --git a/data/ui/app/libs/ng-upload/ng-upload.js b/data/ui/app/libs/ng-upload/ng-upload.js index fae7a44..cedaa1f 100644 --- a/data/ui/app/libs/ng-upload/ng-upload.js +++ b/data/ui/app/libs/ng-upload/ng-upload.js @@ -38,7 +38,7 @@ angular.module('ngUpload', []) options.enableControls = attrs.uploadOptionsEnableControls; // submit the form - requires jQuery - var form = element.parents('form[ng-upload]') || element.parents('form.ng-upload'); + var form = $(element).parents('form[ng-upload]') || $(element).parents('form.ng-upload'); // Retrieve the callback function var fn = $parse(attrs.uploadSubmit); @@ -104,4 +104,4 @@ angular.module('ngUpload', []) element.attr("encoding", "multipart/form-data"); } }; - }]); \ No newline at end of file + }]);

1 0

[ooni-probe/master] Fix bug in HTTP api
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 21ae4fb974e3eea55929e1613d29d1675fe4e03a Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 17:45:19 2013 +0200 Fix bug in HTTP api --- ooni/api/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/api/spec.py b/ooni/api/spec.py index f707253..ecd514c 100644 --- a/ooni/api/spec.py +++ b/ooni/api/spec.py @@ -164,7 +164,7 @@ class StartTest(ORequestHandler): test_file = oonidApplication.director.netTests[test_name]['path'] test_options = json.loads(self.request.body) tmp_files = [] - if (test_options['manual_input']): + if ('manual_input' in test_options): for option, content in test_options['manual_input'].items(): fd, path = write_temporary_input(content) test_options[option] = path

1 0

[ooni-probe/master] Add oonid script to run ooni daemon
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 22c0c16f8eb9b7fa5680e8f23167ded4a7082835 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jun 3 17:56:11 2013 +0200 Add oonid script to run ooni daemon --- bin/oonid | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/oonid b/bin/oonid new file mode 100755 index 0000000..dd59eb9 --- /dev/null +++ b/bin/oonid @@ -0,0 +1,7 @@ +#!/bin/bash +# XXX This is very Ghetto. The proper way to do this is to make a twisted +# plugin. The plugin will then be installed and can then be run directly with +# `twistd oonid` + +OONID_PATH=`python -c 'import ooni;import os;print os.path.join(os.path.dirname(os.path.abspath(ooni.__file__)), "oonid.py")'` +twistd -y $OONID_PATH

1 0

[ooni-probe/master] Merge branch 'master' into feature/ui
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 2e0da24fb07ac2b6b8669926b8c14161b5406733 Merge: 21ae4fb 3f9035f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 17:49:40 2013 +0200 Merge branch 'master' into feature/ui * master: Only process the input when we are dealing with URLs Remove really old Makefile which seems to no longer point to valid directories. Remove oonib report archival script. Remove line to ignore oonib/oonibackend.conf from .gitignore. Remove log factory for oonib from ooni/utils/log.py. Remove old bin/oonib script. Stop reactor when we can't start the sniffer. Strip superflous data from ASN number Fix import error for pushFilenameStack Add Dominic Hamon's nettest for running tests written with other interpreters. Parsley 1.1 is now in pypi so we don't have to mirror it. Remove unused geodata module that depended on pygeoip Explicit versions of scapy, parsley and dnspython Remove oonib from ooni-probe repo. Update requirements.txt to what twisted wants. Include by default the details of the probe Add extra commands to step 1 Add URL inputProcessor to tcpconnect Add inputProcessor to http_host Set pcap name for each NetTest, and use consistent timestamps. Conflicts: ooni/config.py ooni/director.py ooni/nettest.py ooni/utils/geodata.py requirements.txt .gitignore | 1 - Makefile | 202 ------------------------------- README.md | 2 + data/nettests/blocking/tcpconnect.py | 24 ++++ data/nettests/manipulation/http_host.py | 10 ++ data/ooniprobe.conf.sample | 8 +- nettests/experimental/script.py | 90 ++++++++++++++ ooni/director.py | 21 +++- ooni/geoip.py | 2 +- ooni/nettest.py | 4 +- ooni/otime.py | 2 + ooni/reporter.py | 2 +- ooni/utils/geodata.py | 40 ------ ooni/utils/log.py | 15 --- requirements.txt | 23 ++-- scripts/archive_reports.py | 174 -------------------------- 16 files changed, 162 insertions(+), 458 deletions(-) diff --cc data/nettests/blocking/tcpconnect.py index 3b22427,0000000..5b432e0 mode 100644,000000..100644 --- a/data/nettests/blocking/tcpconnect.py +++ b/data/nettests/blocking/tcpconnect.py @@@ -1,45 -1,0 +1,69 @@@ +# -*- encoding: utf-8 -*- +from twisted.internet.protocol import Factory, Protocol +from twisted.internet.endpoints import TCP4ClientEndpoint + +from twisted.internet.error import ConnectionRefusedError +from twisted.internet.error import TCPTimedOutError, TimeoutError + +from ooni import nettest +from ooni.errors import handleAllFailures +from ooni.utils import log + +class TCPFactory(Factory): + def buildProtocol(self, addr): + return Protocol() + +class TCPConnectTest(nettest.NetTestCase): + name = "TCP Connect" + author = "Arturo Filastò" + version = "0.1" + inputFile = ['file', 'f', None, + 'File containing the IP:PORT combinations to be tested, one per line'] + + requiredOptions = ['file'] + def test_connect(self): + """ + This test performs a TCP connection to the remote host on the specified port. + the report will contains the string 'success' if the test has + succeeded, or the reason for the failure if it has failed. + """ + host, port = self.input.split(":") + def connectionSuccess(protocol): + protocol.transport.loseConnection() + log.debug("Got a connection to %s" % self.input) + self.report["connection"] = 'success' + + def connectionFailed(failure): + self.report['connection'] = handleAllFailures(failure) + + from twisted.internet import reactor + point = TCP4ClientEndpoint(reactor, host, int(port)) + d = point.connect(TCPFactory()) + d.addCallback(connectionSuccess) + d.addErrback(connectionFailed) + return d + ++ def inputProcessor(self, filename=None): ++ """ ++ This inputProcessor extracts name:port pairs from urls ++ XXX: Does not support unusual port numbers ++ """ ++ def strip_url(address): ++ proto, path = x.strip().split('://') ++ proto = proto.lower() ++ host = path.split('/')[0] ++ if proto == 'http': ++ return "%s:80" % host ++ if proto == 'https': ++ return "%s:443" % host ++ ++ if filename: ++ fp = open(filename) ++ for x in fp.readlines(): ++ if x.startswith("http"): ++ yield strip_url(x) ++ else: ++ yield x.strip() ++ fp.close() ++ else: ++ pass diff --cc data/nettests/manipulation/http_host.py index d95d836,0000000..3f1e0c6 mode 100644,000000..100644 --- a/data/nettests/manipulation/http_host.py +++ b/data/nettests/manipulation/http_host.py @@@ -1,141 -1,0 +1,151 @@@ +# -*- encoding: utf-8 -*- +# +# HTTP Host Test +# ************** +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import json +from twisted.python import usage + +from ooni.utils import randomStr, randomSTR + +from ooni.utils import log +from ooni.templates import httpt + +class UsageOptions(usage.Options): + optParameters = [['backend', 'b', 'http://127.0.0.1:57001', + 'URL of the test backend to use. Should be \ + listening on port 80 and be a \ + HTTPReturnJSONHeadersHelper'], + ['content', 'c', None, 'The file to read \ + from containing the content of a block page']] + +class HTTPHost(httpt.HTTPTest): + """ + This test is aimed at detecting the presence of a transparent HTTP proxy + and enumerating the sites that are being censored by it. + + It places inside of the Host header field the hostname of the site that is + to be tested for censorship and then determines if the probe is behind a + transparent HTTP proxy (because the response from the backend server does + not match) and if the site is censorsed, by checking if the page that it + got back matches the input block page. + """ + name = "HTTP Host" + author = "Arturo Filastò" + version = "0.2.3" + + randomizeUA = False + usageOptions = UsageOptions + + inputFile = ['file', 'f', None, + 'List of hostnames to test for censorship'] + + requiredOptions = ['backend'] + + def test_filtering_prepend_newline_to_method(self): + headers = {} + headers["Host"] = [self.input] + return self.doRequest(self.localOptions['backend'], method="\nGET", + headers=headers) + + def test_filtering_add_tab_to_host(self): + headers = {} + headers["Host"] = [self.input + '\t'] + return self.doRequest(self.localOptions['backend'], + headers=headers) + + def test_filtering_of_subdomain(self): + headers = {} + headers["Host"] = [randomStr(10) + '.' + self.input] + return self.doRequest(self.localOptions['backend'], + headers=headers) + + def test_filtering_via_fuzzy_matching(self): + headers = {} + headers["Host"] = [randomStr(10) + self.input + randomStr(10)] + return self.doRequest(self.localOptions['backend'], + headers=headers) + + def test_send_host_header(self): + """ + Stuffs the HTTP Host header field with the site to be tested for + censorship and does an HTTP request of this kind to our backend. + + We randomize the HTTP User Agent headers. + """ + headers = {} + headers["Host"] = [self.input] + return self.doRequest(self.localOptions['backend'], + headers=headers) + + def check_for_censorship(self, body): + """ + If we have specified what a censorship page looks like here we will + check if the page we are looking at matches it. + + XXX this is not tested, though it is basically what was used to detect + censorship in the palestine case. + """ + if self.localOptions['content']: + self.report['censored'] = True + censorship_page = open(self.localOptions['content']) + response_page = iter(body.split("\n")) + + for censorship_line in censorship_page.xreadlines(): + response_line = response_page.next() + if response_line != censorship_line: + self.report['censored'] = False + break + + censorship_page.close() + else: + self.report['censored'] = None + + def processResponseBody(self, body): + """ + XXX this is to be filled in with either a domclass based classified or + with a rule that will allow to detect that the body of the result is + that of a censored site. + """ + # If we don't see a json array we know that something is wrong for + # sure + if not body.startswith("{"): + log.msg("This does not appear to be JSON") + self.report['transparent_http_proxy'] = True + self.check_for_censorship(body) + return + try: + content = json.loads(body) + except: + log.msg("The json does not parse, this is not what we expected") + self.report['transparent_http_proxy'] = True + self.check_for_censorship(body) + return + + # We base the determination of the presence of a transparent HTTP + # proxy on the basis of the response containing the json that is to be + # returned by a HTTP Request Test Helper + if 'request_headers' in content and \ + 'request_line' in content and \ + 'headers_dict' in content: + log.msg("Found the keys I expected in %s" % content) + self.report['transparent_http_proxy'] = False + self.report['censored'] = False + else: + log.msg("Did not find the keys I expected in %s" % content) + self.report['transparent_http_proxy'] = True + self.check_for_censorship(body) + ++ def inputProcessor(self, filename=None): ++ """ ++ This inputProcessor extracts domain names from urls ++ """ ++ if filename: ++ fp = open(filename) ++ for x in fp.readlines(): ++ yield x.strip().split('//')[-1].split('/')[0] ++ fp.close() ++ else: pass diff --cc data/ooniprobe.conf.sample index a12b526,0000000..ca94c86 mode 100644,000000..100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@@ -1,55 -1,0 +1,55 @@@ +# This is the configuration file for OONIProbe +# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html +# Keep in mind that indentation matters. + +basic: + # Where OONIProbe should be writing it's log file + logfile: /var/log/ooniprobe.log +privacy: + # Should we include the IP address of the probe in the report? - includeip: false ++ includeip: true + # Should we include the ASN of the probe in the report? - includeasn: false ++ includeasn: true + # Should we include the country as reported by GeoIP in the report? - includecountry: false ++ includecountry: true + # Should we include the city as reported by GeoIP in the report? - includecity: false ++ includecity: true + # Should we collect a full packet capture on the client? + includepcap: false +reports: + # This is a packet capture file (.pcap) to load as a test: + pcap: null + collector: 'httpo://nkvphnp3p6agi5qq.onion' +advanced: + geoip_data_dir: /usr/share/ooni/ + debug: true + # enable if auto detection fails + #tor_binary: /usr/sbin/tor + # For auto detection + interface: auto + # Of specify a specific interface + #interface: wlan0 + # If you do not specify start_tor, you will have to have Tor running and + # explicitly set the control port and SOCKS port + start_tor: false + # After how many seconds we should give up on a particular measurement + measurement_timeout: 30 + # After how many retries we should give up on a measurement + measurement_retries: 2 + # How many measurments to perform concurrently + measurement_concurrency: 100 + # After how may seconds we should give up reporting + reporting_timeout: 30 + # After how many retries to give up on reporting + reporting_retries: 3 + # How many reports to perform concurrently + reporting_concurrency: 20 +tor: + socks_port: 9050 + #control_port: 9051 + # Specify the absolute path to the Tor bridges to use for testing + #bridges: bridges.list + # Specify path of the tor datadirectory. + # This should be set to something to avoid having Tor download each time + # the descriptors and consensus data. + #data_dir: ~/.tor/ diff --cc ooni/director.py index bb02201,809eae4..30a68f9 --- a/ooni/director.py +++ b/ooni/director.py @@@ -6,10 -6,10 +6,10 @@@ import r from ooni import geoip from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report - from ooni.utils import log, checkForRoot + from ooni.utils import log, checkForRoot, pushFilenameStack from ooni.utils.net import randomFreePort -from ooni.nettest import NetTest +from ooni.nettest import NetTest, getNetTestInformation +from ooni.settings import config from ooni import errors from txtorcon import TorConfig @@@ -87,39 -86,10 +87,40 @@@ class Director(object) # This deferred is fired once all the measurements and their reporting # tasks are completed. self.allTestsDone = defer.Deferred() + self.sniffer = None + def getNetTests(self): + nettests = {} + def is_nettest(filename): + return not filename == '__init__.py' \ + and filename.endswith('.py') + + for category in self.categories: + dirname = os.path.join(config.nettest_directory, category) + # print path to all filenames. + for filename in os.listdir(dirname): + if is_nettest(filename): + net_test_file = os.path.join(dirname, filename) + nettest = getNetTestInformation(net_test_file) + + if nettest['id'] in nettests: + log.err("Found a two tests with the same name %s, %s" % + (nettest_path, nettests[nettest['id']]['path'])) + else: + category = dirname.replace(config.nettest_directory, '') + nettests[nettest['id']] = nettest + + return nettests + @defer.inlineCallbacks def start(self): + self.netTests = self.getNetTests() + + if config.privacy.includepcap: + log.msg("Starting") + if not config.reports.pcap: + config.generate_pcap_filename() + self.startSniffing() if config.advanced.start_tor: log.msg("Starting Tor...") diff --cc ooni/reporter.py index 621eaeb,85df36e..251e824 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -29,10 -29,10 +29,10 @@@ except ImportError from ooni import errors from ooni import otime - from ooni.utils import geodata, pushFilenameStack + from ooni.utils import pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents -from ooni import config +from ooni.settings import config from ooni.tasks import ReportEntry, TaskTimedOut, ReportTracker

1 0