June 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Remove oonib related data from ooniprobe
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 42b43f0a8ae841546f00f49e1b26b511e42ec1b0 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Apr 25 16:05:01 2013 +0200 Remove oonib related data from ooniprobe --- bin/oonib | 36 ------- oonib/INSTALL | 4 - oonib/README.md | 117 ---------------------- oonib/__init__.py | 24 ----- oonib/config.py | 58 ----------- oonib/db/__init__.py | 30 ------ oonib/db/tables.py | 123 ----------------------- oonib/models.py | 129 ------------------------- oonib/oonibackend.py | 74 -------------- oonib/report/__init__.py | 5 - oonib/report/api.py | 106 -------------------- oonib/report/file_collector.py | 193 ------------------------------------- oonib/requirements.txt | 24 ----- oonib/runner.py | 81 ---------------- oonib/testhelpers/__init__.py | 5 - oonib/testhelpers/dns_helpers.py | 16 --- oonib/testhelpers/http_helpers.py | 154 ----------------------------- oonib/testhelpers/ssl_helpers.py | 9 -- oonib/testhelpers/tcp_helpers.py | 72 -------------- 19 files changed, 1260 deletions(-) diff --git a/bin/oonib b/bin/oonib deleted file mode 100755 index 79885ba..0000000 --- a/bin/oonib +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python - -import sys -import os -# Avoid the export PYTHONPATH insanity -sys.path[:] = map(os.path.abspath, sys.path) -this_directory = os.path.dirname(__file__) -root = os.path.abspath(os.path.join(this_directory, '..')) -sys.path.insert(0, root) - -backend_script = os.path.join(root, 'oonib', 'oonibackend.py') - -from twisted.python import log, usage -from twisted.internet import reactor -from twisted.application import app - -from oonib import runner -from oonib.oonibackend import application - -sys.argv[1:] = ['-ny', backend_script] - -# Uncomment this line to daemonize -#sys.argv[1:] = ['-y', backend_script] - -def runApp(config): - runner.OBaseRunner(config).run() - -config = runner.ServerOptions() -try: - config.parseOptions() -except usage.error, ue: - print config - print "%s: %s" % (sys.argv[0], ue) -else: - runApp(config) - diff --git a/oonib/INSTALL b/oonib/INSTALL deleted file mode 100644 index 1622707..0000000 --- a/oonib/INSTALL +++ /dev/null @@ -1,4 +0,0 @@ -BEWARE: This requires python 2.7.3 -storm (Storm ORM) -transaction (zope transaction) - diff --git a/oonib/README.md b/oonib/README.md deleted file mode 100644 index d11f876..0000000 --- a/oonib/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# Dependencies - -The extra dependencies necessary to run OONIB are: - -* twisted-names -* cyclone: https://github.com/fiorix/cyclone - -We recommend that you use a python virtualenv. See OONI's README.md. - -# Generate self signed certs for OONIB - -If you want to use the HTTPS test helper, you will need to create a certificate: - - openssl genrsa -des3 -out private.key 4096 - openssl req -new -key private.key -out server.csr - cp private.key private.key.org - # Remove passphrase from key - openssl rsa -in private.key.org -out private.key - openssl x509 -req -days 365 -in server.csr -signkey private.key -out certificate.crt - rm private.key.org - -Don't forget to update oonib/config.py options helpers.ssl.private_key and -helpers.ssl.certificate - -# Redirect low ports with iptables - -The following iptables commands will map connections on low ports to those bound by oonib - - # Map port 80 to config.helpers.http_return_request.port (default: 57001) - iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 57001 - # Map port 443 to config.helpers.ssl.port (default: 57006) - iptables -t nat -A PREROUTING -p tcp -m tcp --dport 443 -j REDIRECT --to-ports 57006 - # Map port 53 udp to config.helpers.dns.udp_port (default: 57004) - iptables -t nat -A PREROUTING -p tcp -m udp --dport 53 -j REDIRECT --tor-ports - # Map port 53 tcp to config.helpers.dns.tcp_port (default: 57005) - iptables -t nat -A PREROUTING -p tcp -m tcp --dport 53 -j REDIRECT --tor-ports - -# Install Tor (Debian). - -You will need a Tor binary on your system. For complete instructions, see also: - - https://www.torproject.org/docs/tor-doc-unix.html.en - https://www.torproject.org/docs/rpms.html.en - -Add this line to your /etc/apt/sources.list, replacing <DISTRIBUTION> -where appropriate: - - deb http://deb.torproject.org/torproject.org <DISTRIBUTION> main - -Add the Tor Project gpg key to apt: - - gpg --keyserver keys.gnupg.net --recv 886DDD89 - gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | sudo apt-key add - - # Update apt and install the torproject keyring, tor, and geoipdb - apt-get update - apt-get install deb.torproject.org-keyring tor tor-geoipdb - -# Update ooni-probe/oonib/config.py - - Set config.main.tor_binary to your Tor path - Set config.main.tor2webmode = False - -# (For Experts Only) Tor2webmode: - -WARNING: provides no anonymity! Use only if you know what you are doing! -Tor2webmode will improve the performance of the collector Hidden Service -by discarding server-side anonymity. - -You will need to build Tor from source. At the time of writing, the latest stable Tor is tor-0.2.3.25. You should use the most recent stable Tor. - -Example: - - git clone https://git.torproject.org/tor.git - git checkout tor-0.2.3.25 - git verify-tag -v tor-0.2.3.25 - -You should see: - - object 17c24b3118224d6536c41fa4e1493a831fb29f0a - type commit - tag tor-0.2.3.25 - tagger Roger Dingledine <arma(a)torproject.org> 1353399116 -0500 - - tag 0.2.3.25 - gpg: Signature made Tue 20 Nov 2012 08:11:59 AM UTC using RSA key ID 19F78451 - gpg: Good signature from "Roger Dingledine <arma(a)mit.edu>" - gpg: aka "Roger Dingledine <arma(a)freehaven.net>" - gpg: aka "Roger Dingledine <arma(a)torproject.org>" - -It is always good idea to verify. - - gpg --fingerprint 19F78451 - pub 4096R/19F78451 2010-05-07 - Key fingerprint = F65C E37F 04BA 5B36 0AE6 EE17 C218 5258 19F7 8451 - uid Roger Dingledine <arma(a)mit.edu> - uid Roger Dingledine <arma(a)freehaven.net> - uid Roger Dingledine <arma(a)torproject.org> - sub 4096R/9B11185C 2012-05-02 [expires: 2013-05-02] - -Build Tor with enable-tor2web-mode - - ./autogen.sh ; ./configure --enable-tor2web-mode ; make - -Copy the tor binary from src/or/tor somewhere and set the corresponding -options in oonib/config.py - -# To launch oonib on system boot - -To launch oonib on startup, you may want to use supervisord (www.supervisord.org) -The following supervisord config will use the virtual environment in -/home/ooni/venv_oonib and start oonib on boot: - - [program:oonib] - command=/home/ooni/venv_oonib/bin/python /home/ooni/ooni-probe/bin/oonib - autostart=true - user=oonib - directory=/home/oonib/ diff --git a/oonib/__init__.py b/oonib/__init__.py deleted file mode 100644 index ab7419c..0000000 --- a/oonib/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE -""" -In here we shall keep track of all variables and objects that should be -instantiated only once and be common to pieces of GLBackend code. -""" -__all__ = ['database', 'db_threadpool'] - -from twisted.python.threadpool import ThreadPool - -from storm.uri import URI -from storm.twisted.transact import Transactor -from storm.databases.sqlite import SQLite - -__version__ = '0.0.1' - -from oonib import config - -database = SQLite(URI(config.main.database_uri)) -db_threadpool = ThreadPool(0, config.main.db_threadpool_size) -db_threadpool.start() -transactor = Transactor(db_threadpool) diff --git a/oonib/config.py b/oonib/config.py deleted file mode 100644 index 8598e00..0000000 --- a/oonib/config.py +++ /dev/null @@ -1,58 +0,0 @@ -from ooni.utils import Storage -import os - -def get_root_path(): - this_directory = os.path.dirname(__file__) - root = os.path.join(this_directory, '..') - root = os.path.abspath(root) - return root - -backend_version = '0.0.1' - -# XXX convert this to something that is a proper config file -main = Storage() - -# This is the location where submitted reports get stored -main.report_dir = os.path.join(get_root_path(), 'oonib', 'reports') - -# This is where tor will place it's Hidden Service hostname and Hidden service -# private key -main.tor_datadir = os.path.join(get_root_path(), 'oonib', 'data', 'tor') - -main.database_uri = "sqlite:"+get_root_path()+"oonib_test_db.db" -main.db_threadpool_size = 10 -#main.tor_binary = '/usr/sbin/tor' -main.tor_binary = '/usr/local/bin/tor' - -# This requires compiling Tor with tor2web mode enabled -# BEWARE!! THIS PROVIDES NO ANONYMITY!! -# ONLY DO IT IF YOU KNOW WHAT YOU ARE DOING!! -# HOSTING A COLLECTOR WITH TOR2WEB MODE GIVES YOU NO ANONYMITY!! -main.tor2webmode = True - -helpers = Storage() - -helpers.http_return_request = Storage() -helpers.http_return_request.port = 57001 -# XXX this actually needs to be the advertised Server HTTP header of our web -# server -helpers.http_return_request.server_version = "Apache" - -helpers.tcp_echo = Storage() -helpers.tcp_echo.port = 57002 - -helpers.daphn3 = Storage() -#helpers.daphn3.yaml_file = "/path/to/data/oonib/daphn3.yaml" -#helpers.daphn3.pcap_file = "/path/to/data/server.pcap" -helpers.daphn3.port = 57003 - -helpers.dns = Storage() -helpers.dns.udp_port = 57004 -helpers.dns.tcp_port = 57005 - -helpers.ssl = Storage() -#helpers.ssl.private_key = /path/to/data/private.key -#helpers.ssl.certificate = /path/to/data/certificate.crt -#helpers.ssl.port = 57006 - - diff --git a/oonib/db/__init__.py b/oonib/db/__init__.py deleted file mode 100644 index 8ddaff0..0000000 --- a/oonib/db/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -__all__ = ['createTables', 'database', 'transactor'] - -from twisted.python.threadpool import ThreadPool -from twisted.internet.defer import inlineCallbacks, returnValue, Deferred - -from storm.locals import Store -from storm.uri import URI -from storm.databases.sqlite import SQLite - -from oonib import database, transactor -from ooni.utils import log - -@inlineCallbacks -def createTables(): - """ - XXX this is to be refactored and only exists for experimentation. - """ - from oonib.db import models - for model_name in models.__all__: - try: - model = getattr(m, model_name) - except Exception, e: - log.err("Error in db initting") - log.err(e) - try: - log.debug("Creating %s" % model) - yield tables.runCreateTable(model, transactor, database) - except Exception, e: - log.debug(str(e)) - diff --git a/oonib/db/tables.py b/oonib/db/tables.py deleted file mode 100644 index 908a295..0000000 --- a/oonib/db/tables.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from twisted.internet.defer import inlineCallbacks - -from storm.locals import Store -from storm.properties import PropertyColumn -from storm.exceptions import StormError - -from storm.variables import BoolVariable, DateTimeVariable, DateVariable -from storm.variables import DecimalVariable, EnumVariable -from storm.variables import FloatVariable, IntVariable, RawStrVariable -from storm.variables import UnicodeVariable, JSONVariable, PickleVariable - -def variableToSQLite(var_type): - """ - We take as input a storm.variable and we output the SQLite string it - represents. - """ - sqlite_type = "VARCHAR" - if isinstance(var_type, BoolVariable): - sqlite_type = "INTEGER" - elif isinstance(var_type, DateTimeVariable): - pass - sqlite_type = "" - elif isinstance(var_type, DateVariable): - pass - elif isinstance(var_type, DecimalVariable): - pass - elif isinstance(var_type, EnumVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, FloatVariable): - sqlite_type = "REAL" - elif isinstance(var_type, IntVariable): - sqlite_type = "INTEGER" - elif isinstance(var_type, RawStrVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, UnicodeVariable): - pass - elif isinstance(var_type, JSONVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, PickleVariable): - sqlite_type = "BLOB" - return "%s" % sqlite_type - -def varsToParametersSQLite(variables, primary_keys): - """ - Takes as input a list of variables (convered to SQLite syntax and in the - form of strings) and primary_keys. - Outputs these variables converted into paramter syntax for SQLites. - - ex. - variables: ["var1 INTEGER", "var2 BOOL", "var3 INTEGER"] - primary_keys: ["var1"] - - output: "(var1 INTEGER, var2 BOOL, var3 INTEGER PRIMARY KEY (var1))" - """ - params = "(" - for var in variables[:-1]: - params += "%s %s, " % var - if len(primary_keys) > 0: - params += "%s %s, " % variables[-1] - params += "PRIMARY KEY (" - for key in primary_keys[:-1]: - params += "%s, " % key - params += "%s))" % primary_keys[-1] - else: - params += "%s %s)" % variables[-1] - return params - -def generateCreateQuery(model): - """ - This takes as input a Storm model and outputs the creation query for it. - """ - query = "CREATE TABLE "+ model.__storm_table__ + " " - - variables = [] - primary_keys = [] - - for attr in dir(model): - a = getattr(model, attr) - if isinstance(a, PropertyColumn): - var_stype = a.variable_factory() - var_type = variableToSQLite(var_stype) - name = a.name - variables.append((name, var_type)) - if a.primary: - primary_keys.append(name) - - query += varsToParametersSQLite(variables, primary_keys) - return query - -def createTable(model, transactor, database): - """ - Create the table for the specified model. - Specification of a transactor and database is useful in unittesting. - """ - if not transactor: - from oonib.db import transactor - if not database: - from oonib.db import database - store = Store(database) - create_query = generateCreateQuery(model) - try: - store.execute(create_query) - # XXX trap the specific error that is raised when the table exists - except StormError, e: - print "Failed to create table!" - print e - store.close() - store.commit() - store.close() - -@inlineCallbacks -def runCreateTable(model, transactor=None, database=None): - """ - Runs the table creation query wrapped in a transaction. - Transactions run in a separate thread. - """ - yield transactor.run(createTable, model, transactor, database) - diff --git a/oonib/models.py b/oonib/models.py deleted file mode 100644 index 22567ad..0000000 --- a/oonib/models.py +++ /dev/null @@ -1,129 +0,0 @@ -__all__ = ['Report', 'TestHelperTMP'] -from storm.twisted.transact import transact -from storm.locals import * - -from ooni.utils import randomStr -from oonib import transactor - -def generateReportID(): - """ - Generates a report ID for usage in the database backed oonib collector. - - XXX note how this function is different from the one in report/api.py - """ - report_id = randomStr(100) - return report_id - -class OModel(object): - - transactor = transactor - - def getStore(self): - return Store(database) - - @transact - def create(self, query): - store = Store(database) - store.execute(query) - store.commit() - - @transact - def save(self): - store = getStore() - store.add(self) - store.commit() - -class Report(OModel): - """ - This represents an OONI Report as stored in the database. - - report_id: this is generated by the backend and is used by the client to - reference a previous report and append to it. It should be - treated as a shared secret between the probe and backend. - - software_name: this indicates the name of the software performing the test - (this will default to ooniprobe) - - software_version: this is the version number of the software running the - test. - - test_name: the name of the test on which the report is being created. - - test_version: indicates the version of the test - - progress: what is the current progress of the report. This allows clients - to report event partial reports up to a certain percentage of - progress. Once the report is complete progress will be 100. - - content: what is the content of the report. If the current progress is less - than 100 we should append to the YAML data structure that is - currently stored in such field. - - XXX this is currently not used. - """ - __storm_table__ = 'reports' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\ - "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\ - "progress VARCHAR, content VARCHAR)" - - - id = Int(primary=True) - - report_id = Unicode() - - software_name = Unicode() - software_version = Unicode() - test_name = Unicode() - test_version = Unicode() - progress = Int() - - content = Unicode() - - @transact - def new(report): - store = self.getStore() - - print "Creating new report %s" % report - - report_id = generateReportID() - new_report = models.Report() - new_report.report_id = unicode(report_id) - - new_report.software_name = report['software_name'] - new_report.software_version = report['software_version'] - new_report.test_name = report['test_name'] - new_report.test_version = report['test_version'] - new_report.progress = report['progress'] - - if 'content' in report: - new_report.content = report['content'] - - print "Report: %s" % report - - store.add(new_report) - try: - store.commit() - except: - store.close() - - defer.returnValue({'backend_version': backend_version, 'report_id': - report_id}) - - -class TestHelperTMP(OModel): - __storm_table__ = 'testhelpertmp' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\ - " client_ip VARCHAR, creation_time VARCHAR)" - - id = Int(primary=True) - - report_id = Unicode() - - test_helper = Unicode() - client_ip = Unicode() - - creation_time = Date() diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py deleted file mode 100644 index 868d7b7..0000000 --- a/oonib/oonibackend.py +++ /dev/null @@ -1,74 +0,0 @@ -# ooni backend -# ************ -# -# This is the backend system responsible for running certain services that are -# useful for censorship detection. -# -# In here we start all the test helpers that are required by ooniprobe and -# start the report collector - -from twisted.application import internet -from twisted.internet import reactor -from twisted.application import internet, service -from twisted.application.service import Application -from twisted.names import dns - -from cyclone import web - -import txtorcon - -from oonib.testhelpers import dns_helpers, ssl_helpers -from oonib.testhelpers import http_helpers, tcp_helpers - -from ooni.utils import log - -from oonib import db_threadpool -from oonib import config - -application = service.Application('oonibackend') -serviceCollection = service.IServiceCollection(application) - -if config.helpers.ssl.port: - print "Starting SSL helper on %s" % config.helpers.ssl.port - ssl_helper = internet.SSLServer(int(config.helpers.ssl.port), - http_helpers.HTTPReturnJSONHeadersHelper(), - ssl_helpers.SSLContext(config)) - ssl_helper.setServiceParent(serviceCollection) - -# Start the DNS Server related services -if config.helpers.dns.tcp_port: - print "Starting TCP DNS Helper on %s" % config.helpers.dns.tcp_port - tcp_dns_helper = internet.TCPServer(int(config.helpers.dns.tcp_port), - dns_helpers.DNSTestHelper()) - tcp_dns_helper.setServiceParent(serviceCollection) - -if config.helpers.dns.udp_port: - print "Starting UDP DNS Helper on %s" % config.helpers.dns.udp_port - udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper()) - udp_dns_helper = internet.UDPServer(int(config.helpers.dns.udp_port), - udp_dns_factory) - udp_dns_helper.setServiceParent(serviceCollection) - -# XXX this needs to be ported -# Start the OONI daphn3 backend -if config.helpers.daphn3.port: - print "Starting Daphn3 helper on %s" % config.helpers.daphn3.port - daphn3_helper = internet.TCPServer(int(config.helpers.daphn3.port), - tcp_helpers.Daphn3Server()) - daphn3_helper.setServiceParent(serviceCollection) - - -if config.helpers.tcp_echo.port: - print "Starting TCP echo helper on %s" % config.helpers.tcp_echo.port - tcp_echo_helper = internet.TCPServer(int(config.helpers.tcp_echo.port), - tcp_helpers.TCPEchoHelper()) - tcp_echo_helper.setServiceParent(serviceCollection) - -if config.helpers.http_return_request.port: - print "Starting HTTP return request helper on %s" % config.helpers.http_return_request.port - http_return_request_helper = internet.TCPServer( - int(config.helpers.http_return_request.port), - http_helpers.HTTPReturnJSONHeadersHelper()) - http_return_request_helper.setServiceParent(serviceCollection) - -reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop) diff --git a/oonib/report/__init__.py b/oonib/report/__init__.py deleted file mode 100644 index fcbf220..0000000 --- a/oonib/report/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -class MissingField(Exception): - pass - -class InvalidRequestField(Exception): - pass diff --git a/oonib/report/api.py b/oonib/report/api.py deleted file mode 100644 index b3d529d..0000000 --- a/oonib/report/api.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -/report - -/pcap - -This is the async pcap reporting system. It requires the client to have created -a report already, but can work independently from test progress. - -""" -import random -import string -import json -import re -import os - -from twisted.internet import reactor, defer - -from cyclone import web - -from ooni import otime -from ooni.utils import randomStr - -from oonib import models, config -from oonib.report import file_collector - -def parseUpdateReportRequest(request): - #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") - - # this is the regexp for the reports that include the timestamp - report_id_regexp = re.compile("[a-zA-Z0-9_-]+$") - - # XXX here we are actually parsing a json object that could be quite big. - # If we want this to scale properly we only want to look at the test_id - # field. - # We are also keeping in memory multiple copies of the same object. A lot - # of optimization can be done. - parsed_request = json.loads(request) - try: - report_id = parsed_request['report_id'] - except KeyError: - raise MissingField('report_id') - - if not re.match(report_id_regexp, report_id): - raise InvalidRequestField('report_id') - - return parsed_request - - -class NewReportHandlerDB(web.RequestHandler): - """ - Responsible for creating and updating reports via database. - XXX this is not yet fully implemented. - """ - - @web.asynchronous - @defer.inlineCallbacks - def post(self): - """ - Creates a new report with the input to the database. - XXX this is not yet implemented. - - * Request - - {'software_name': 'XXX', - 'software_version': 'XXX', - 'test_name': 'XXX', - 'test_version': 'XXX', - 'progress': 'XXX', - 'content': 'XXX' - } - - Optional: - 'test_helper': 'XXX' - 'client_ip': 'XXX' - - * Response - - {'backend_version': 'XXX', 'report_id': 'XXX'} - - """ - report_data = json.loads(self.request.body) - new_report = models.Report() - log.debug("Got this request %s" % report_data) - result = yield new_report.new(report_data) - self.write(result) - self.finish() - - def put(self): - """ - Update an already existing report with the database. - - XXX this is not yet implemented. - - {'report_id': 'XXX', - 'content': 'XXX' - } - """ - pass - - -reportingBackendAPI = [ - (r"/report", file_collector.NewReportHandlerFile), - (r"/pcap", file_collector.PCAPReportHandler) -] - -reportingBackend = web.Application(reportingBackendAPI, debug=True) diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py deleted file mode 100644 index 6d5584c..0000000 --- a/oonib/report/file_collector.py +++ /dev/null @@ -1,193 +0,0 @@ -import random -import string -import json -import re -import os - -from twisted.internet import fdesc - -from cyclone import web - -from ooni.utils import randomStr -from ooni import otime - -from oonib.report import MissingField, InvalidRequestField - -from oonib import config - -def parseUpdateReportRequest(request): - #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") - - # this is the regexp for the reports that include the timestamp - report_id_regexp = re.compile("[a-zA-Z0-9_\-]+$") - - # XXX here we are actually parsing a json object that could be quite big. - # If we want this to scale properly we only want to look at the test_id - # field. - # We are also keeping in memory multiple copies of the same object. A lot - # of optimization can be done. - parsed_request = json.loads(request) - try: - report_id = parsed_request['report_id'] - except KeyError: - raise MissingField('report_id') - - if not re.match(report_id_regexp, report_id): - raise InvalidRequestField('report_id') - - return parsed_request - - - -def parseNewReportRequest(request): - """ - Here we parse a new report request. - """ - version_string = re.compile("[0-9A-Za-z_\-\.]+$") - name = re.compile("[a-zA-Z0-9_\- ]+$") - probe_asn = re.compile("AS[0-9]+$") - - expected_request = { - 'software_name': name, - 'software_version': version_string, - 'test_name': name, - 'test_version': version_string, - 'probe_asn': probe_asn - } - - parsed_request = json.loads(request) - if not parsed_request['probe_asn']: - parsed_request['probe_asn'] = 'AS0' - - for k, regexp in expected_request.items(): - try: - value_to_check = parsed_request[k] - except KeyError: - raise MissingField(k) - - print "Matching %s with %s | %s" % (regexp, value_to_check, k) - if re.match(regexp, str(value_to_check)): - continue - else: - raise InvalidRequestField(k) - - return parsed_request - -class NewReportHandlerFile(web.RequestHandler): - """ - Responsible for creating and updating reports by writing to flat file. - """ - - def post(self): - """ - Creates a new report with the input - - * Request - - {'software_name': 'XXX', - 'software_version': 'XXX', - 'test_name': 'XXX', - 'test_version': 'XXX', - 'probe_asn': 'XXX' - 'content': 'XXX' - } - - Optional: - 'test_helper': 'XXX' - 'client_ip': 'XXX' - - (not implemented, nor in client, nor in backend) - The idea behind these two fields is that it would be interesting to - also collect how the request was observed from the collectors point - of view. - - We use as a unique key the client_ip address and a time window. We - then need to tell the test_helper that is selected the client_ip - address and tell it to expect a connection from a probe in that time - window. - - Once the test_helper sees a connection from that client_ip it will - store for the testing session the data that it receives. - When the probe completes the report (or the time window is over) the - final report will include also the data collected from the - collectors view point. - - * Response - - {'backend_version': 'XXX', 'report_id': 'XXX'} - - """ - # XXX here we should validate and sanitize the request - try: - report_data = parseNewReportRequest(self.request.body) - except InvalidRequestField, e: - raise web.HTTPError(400, "Invalid Request Field %s" % e) - except MissingField, e: - raise web.HTTPError(400, "Missing Request Field %s" % e) - - print "Parsed this data %s" % report_data - software_name = report_data['software_name'] - software_version = report_data['software_version'] - test_name = report_data['test_name'] - test_version = report_data['test_version'] - probe_asn = report_data['probe_asn'] - content = report_data['content'] - - if not probe_asn: - probe_asn = "AS0" - - report_id = otime.timestamp() + '_' \ - + probe_asn + '_' \ - + randomStr(50) - - # The report filename contains the timestamp of the report plus a - # random nonce - report_filename = os.path.join(config.main.report_dir, report_id) - - response = {'backend_version': config.backend_version, - 'report_id': report_id - } - - self.writeToReport(report_filename, - report_data['content']) - - self.write(response) - - def writeToReport(self, report_filename, data): - with open(report_filename, 'w+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), data) - - def put(self): - """ - Update an already existing report. - - { - 'report_id': 'XXX', - 'content': 'XXX' - } - """ - parsed_request = parseUpdateReportRequest(self.request.body) - - report_id = parsed_request['report_id'] - - print "Got this request %s" % parsed_request - report_filename = os.path.join(config.main.report_dir, - report_id) - - self.updateReport(report_filename, parsed_request['content']) - - def updateReport(self, report_filename, data): - try: - with open(report_filename, 'a+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), data) - except IOError as e: - web.HTTPError(404, "Report not found") - -class PCAPReportHandler(web.RequestHandler): - def get(self): - pass - - def post(self): - pass diff --git a/oonib/requirements.txt b/oonib/requirements.txt deleted file mode 100644 index da6ec81..0000000 --- a/oonib/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -PyYAML>=3.10 -Pygments>=1.5 -Twisted>=12.2.0 -argparse>=1.2.1 -cyclone>=1.0-rc13 -distribute>=0.6.24 -docutils>=0.9.1 -ipaddr>=2.1.10 -pyOpenSSL>=0.13 -pygeoip>=0.2.5 -# -# This is a Tor Project mirror with valid SSL/TLS certs that is stable and fast -# -# Originally fetched from the hg repo on secdev.org: -# https://hg.secdev.org/scapy/archive/tip.zip#egg=scapy -# Mirrored on Tor's webserver: -https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/scapy-02-25-2013-tip.zip -storm>=0.19 -transaction>=1.3.0 -txtorcon -wsgiref>=0.1.2 -zope.component>=4.0.0 -zope.event>=4.0.0 -zope.interface>=4.0.1 diff --git a/oonib/runner.py b/oonib/runner.py deleted file mode 100644 index 407f10d..0000000 --- a/oonib/runner.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -In here we define a runner for the oonib backend system. -We are just extending the - -""" - -from twisted.internet import reactor -from twisted.application import service, internet, app -from twisted.python.runtime import platformType - -import txtorcon - -from oonib.report.api import reportingBackend - -from oonib import config -from ooni.utils import log - -def txSetupFailed(failure): - log.err("Setup failed") - log.exception(failure) - -def setupCollector(tor_process_protocol): - def setup_complete(port): - print "Exposed collector Tor hidden service on httpo://%s" % port.onion_uri - - torconfig = txtorcon.TorConfig(tor_process_protocol.tor_protocol) - public_port = 80 - # XXX there is currently a bug in txtorcon that prevents data_dir from - # being passed properly. Details on the bug can be found here: - # https://github.com/meejah/txtorcon/pull/22 - hs_endpoint = txtorcon.TCPHiddenServiceEndpoint(reactor, torconfig, - public_port, data_dir=config.main.tor_datadir) - hidden_service = hs_endpoint.listen(reportingBackend) - hidden_service.addCallback(setup_complete) - hidden_service.addErrback(txSetupFailed) - -def startTor(): - def updates(prog, tag, summary): - print "%d%%: %s" % (prog, summary) - - torconfig = txtorcon.TorConfig() - torconfig.SocksPort = 9055 - if config.main.tor2webmode: - torconfig.Tor2webMode = 1 - torconfig.save() - d = txtorcon.launch_tor(torconfig, reactor, - tor_binary=config.main.tor_binary, - progress_updates=updates) - d.addCallback(setupCollector) - d.addErrback(txSetupFailed) - -class OBaseRunner(): - pass - -if platformType == "win32": - from twisted.scripts._twistw import ServerOptions, \ - WindowsApplicationRunner - - OBaseRunner = WindowsApplicationRunner - # XXX Current we don't support windows for the starting of Tor Hidden Service - -else: - from twisted.scripts._twistd_unix import ServerOptions, \ - UnixApplicationRunner - class OBaseRunner(UnixApplicationRunner): - def postApplication(self): - """ - To be called after the application is created: start the - application and run the reactor. After the reactor stops, - clean up PID files and such. - """ - self.startApplication(self.application) - # This is our addition. The rest is taken from - # twisted/scripts/_twistd_unix.py 12.2.0 - startTor() - self.startReactor(None, self.oldstdout, self.oldstderr) - self.removePID(self.config['pidfile']) - -OBaseRunner.loggerFactory = log.LoggerFactory - - diff --git a/oonib/testhelpers/__init__.py b/oonib/testhelpers/__init__.py deleted file mode 100644 index 4dbb547..0000000 --- a/oonib/testhelpers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from . import dns_helpers -from . import http_helpers -from . import tcp_helpers - -__all__ = ['dns_helpers', 'http_helpers', 'tcp_helpers'] diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py deleted file mode 100644 index cb4ff9f..0000000 --- a/oonib/testhelpers/dns_helpers.py +++ /dev/null @@ -1,16 +0,0 @@ -from twisted.internet.protocol import Factory, Protocol -from twisted.internet import reactor -from twisted.names import dns -from twisted.names import client, server - -class DNSTestHelper(server.DNSServerFactory): - def __init__(self, authorities = None, - caches = None, clients = None, - verbose = 0): - resolver = client.Resolver(servers=[('8.8.8.8', 53)]) - server.DNSServerFactory.__init__(self, authorities = authorities, - caches = caches, clients = [resolver], - verbose = verbose) - def handleQuery(self, message, protocol, address): - print message, protocol, address - server.DNSServerFactory.handleQuery(self, message, protocol, address) diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py deleted file mode 100644 index 3a76b9a..0000000 --- a/oonib/testhelpers/http_helpers.py +++ /dev/null @@ -1,154 +0,0 @@ -import json -import random -import string - -from twisted.application import internet, service -from twisted.internet import protocol, reactor, defer -from twisted.protocols import basic -from twisted.web import resource, server, static, http -from twisted.web.microdom import escape - -from cyclone.web import RequestHandler, Application - -from twisted.protocols import policies, basic -from twisted.web.http import Request - -class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): - """ - This is a simplified version of twisted.web.http.HTTPChannel to overcome - header lowercase normalization. It does not actually implement the HTTP - protocol, but only the subset of it that we need for testing. - - What this HTTP channel currently does is process the HTTP Request Line and - the Request Headers and returns them in a JSON datastructure in the order - we received them. - - The returned JSON dict looks like so: - - { - 'request_headers': - [['User-Agent', 'IE6'], ['Content-Length', 200]] - 'request_line': - 'GET / HTTP/1.1' - } - """ - requestFactory = Request - __first_line = 1 - __header = '' - __content = None - - length = 0 - maxHeaders = 500 - requestLine = '' - headers = [] - - timeOut = 60 * 60 * 12 - - def __init__(self): - self.requests = [] - - def connectionMade(self): - self.setTimeout(self.timeOut) - - def lineReceived(self, line): - if self.__first_line: - self.requestLine = line - self.__first_line = 0 - elif line == '': - # We have reached the end of the headers. - if self.__header: - self.headerReceived(self.__header) - self.__header = '' - self.allHeadersReceived() - self.setRawMode() - elif line[0] in ' \t': - # This is to support header field value folding over multiple lines - # as specified by rfc2616. - self.__header = self.__header+'\n'+line - else: - if self.__header: - self.headerReceived(self.__header) - self.__header = line - - def headerReceived(self, line): - try: - header, data = line.split(':', 1) - self.headers.append((header, data.strip())) - except: - log.err("Got malformed HTTP Header request field") - log.err("%s" % line) - - def allHeadersReceived(self): - headers_dict = {} - for k, v in self.headers: - if k not in headers_dict: - headers_dict[k] = [] - headers_dict[k].append(v) - - response = {'request_headers': self.headers, - 'request_line': self.requestLine, - 'headers_dict': headers_dict - } - json_response = json.dumps(response) - self.transport.write('HTTP/1.1 200 OK\r\n\r\n') - self.transport.write('%s' % json_response) - self.transport.loseConnection() - - -class HTTPReturnJSONHeadersHelper(protocol.ServerFactory): - protocol = SimpleHTTPChannel - def buildProtocol(self, addr): - p = self.protocol() - p.headers = [] - return p - -class HTTPTrapAll(RequestHandler): - def _execute(self, transforms, *args, **kwargs): - self._transforms = transforms - defer.maybeDeferred(self.prepare).addCallbacks( - self._execute_handler, - lambda f: self._handle_request_exception(f.value), - callbackArgs=(args, kwargs)) - - def _execute_handler(self, r, args, kwargs): - if not self._finished: - args = [self.decode_argument(arg) for arg in args] - kwargs = dict((k, self.decode_argument(v, name=k)) - for (k, v) in kwargs.iteritems()) - # This is where we do the patching - # XXX this is somewhat hackish - d = defer.maybeDeferred(self.all, *args, **kwargs) - d.addCallbacks(self._execute_success, self._execute_failure) - self.notifyFinish().addCallback(self.on_connection_close) - - -class HTTPRandomPage(HTTPTrapAll): - """ - This generates a random page of arbitrary length and containing the string - selected by the user. - /<length>/<keyword> - XXX this is currently disabled as it is not of use to any test. - """ - isLeaf = True - def _gen_random_string(self, length): - return ''.join(random.choice(string.letters) for x in range(length)) - - def genRandomPage(self, length=100, keyword=None): - data = self._gen_random_string(length/2) - if keyword: - data += keyword - data += self._gen_random_string(length - length/2) - data += '\n' - return data - - def all(self, length, keyword): - length = 100 - if length > 100000: - length = 100000 - return self.genRandomPage(length, keyword) - -HTTPRandomPageHelper = Application([ - # XXX add regexps here - (r"/(.*)/(.*)", HTTPRandomPage) -]) - diff --git a/oonib/testhelpers/ssl_helpers.py b/oonib/testhelpers/ssl_helpers.py deleted file mode 100644 index 5c74996..0000000 --- a/oonib/testhelpers/ssl_helpers.py +++ /dev/null @@ -1,9 +0,0 @@ -from twisted.internet import ssl -from oonib import config - -class SSLContext(ssl.DefaultOpenSSLContextFactory): - def __init__(self, *args, **kw): - ssl.DefaultOpenSSLContextFactory.__init__(self, - config.helpers.ssl.private_key, - config.helpers.ssl.certificate) - diff --git a/oonib/testhelpers/tcp_helpers.py b/oonib/testhelpers/tcp_helpers.py deleted file mode 100644 index 4d32ae0..0000000 --- a/oonib/testhelpers/tcp_helpers.py +++ /dev/null @@ -1,72 +0,0 @@ - -from twisted.internet.protocol import Protocol, Factory, ServerFactory -from twisted.internet.error import ConnectionDone - -from oonib import config -from ooni.utils import log -from ooni.kit.daphn3 import Daphn3Protocol -from ooni.kit.daphn3 import read_pcap, read_yaml - -class TCPEchoProtocol(Protocol): - def dataReceived(self, data): - self.transport.write(data) - -class TCPEchoHelper(Factory): - """ - A very simple echo protocol implementation - """ - protocol = TCPEchoProtocol - -if config.helpers.daphn3.yaml_file: - daphn3Steps = read_pcap(config.helpers.daphn3.yaml_file) - -elif config.helpers.daphn3.pcap_file: - daphn3Steps = read_yaml(config.helpers.daphn3.pcap_file) - -else: - daphn3Steps = [{'client': 'client_packet'}, - {'server': 'server_packet'}] - -class Daphn3ServerProtocol(Daphn3Protocol): - def nextStep(self): - log.debug("Moving on to next step in the state walk") - self.current_data_received = 0 - # Python why? - if self.current_step >= (len(self.steps) - 1): - log.msg("Reached the end of the state machine") - log.msg("Censorship fingerpint bisected!") - step_idx, mutation_idx = self.factory.mutation - log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx)) - #self.transport.loseConnection() - if self.report: - self.report['mutation_idx'] = mutation_idx - self.report['step_idx'] = step_idx - return - else: - self.current_step += 1 - if self._current_step_role() == self.role: - # We need to send more data because we are again responsible for - # doing so. - self.sendPayload() - -class Daphn3Server(ServerFactory): - """ - This is the main class that deals with the daphn3 server side component. - We keep track of global state of every client here. - Every client is identified by their IP address and the state of mutation is - stored by using their IP address as a key. This may lead to some bugs if - two different clients are sharing the same IP, but hopefully the - probability of such thing is not that likely. - """ - protocol = Daphn3ServerProtocol - # step_idx, mutation_idx - mutation = [0, 0] - def buildProtocol(self, addr): - p = self.protocol() - p.steps = daphn3Steps - p.role = "server" - p.factory = self - return p - - -

1 0

[ooni-probe/master] Merge branch 'feature/task_manager' into feature/ui
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 81c53bbc6107d955731eabe6d41e24c586d07122 Merge: 3b88935 056a24b Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 29 18:15:47 2013 +0200 Merge branch 'feature/task_manager' into feature/ui * feature/task_manager: Make how we keep track of successful reports more robust. ooni/reporter.py | 12 ++++++------ ooni/tasks.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --cc ooni/reporter.py index 1095cb1,732d516..5d8f92b --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -32,9 -32,9 +32,9 @@@ from ooni import otim from ooni.utils import geodata, pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents -from ooni import config +from ooni.settings import config - from ooni.tasks import ReportEntry, TaskTimedOut + from ooni.tasks import ReportEntry, TaskTimedOut, ReportTracker class ReporterException(Exception): pass diff --cc ooni/tasks.py index f686a9c,428fdbf..35f1236 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@@ -129,7 -129,28 +129,26 @@@ class Measurement(TaskWithTimeout) d = self.netTestMethod() return d + class ReportTracker(object): + def __init__(self, reporters): + self.report_completed = 0 + self.reporters = reporters + + def finished(self): + """ + Returns true if all the tasks are done. False if not. + """ + if self.report_completed == len(self.reporters): + return True + return False + + def completed(self): + """ + Called when a new report is completed. + """ + self.report_completed += 1 + class ReportEntry(TaskWithTimeout): - timeout = config.advanced.reporting_timeout - def __init__(self, reporter, measurement): self.reporter = reporter self.measurement = measurement

1 0

[ooni-probe/master] Implement in-browser input file selection views
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 13a65fd6adb798f6c2bdcc4584a5cff52e694d7c Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 29 18:54:01 2013 +0200 Implement in-browser input file selection views This will allow the user to copy and paste the input file into a textarea. --- data/ui/app/scripts/controllers.js | 28 ++++++++++++++++++++++++++++ data/ui/app/views/test.html | 12 +++++++++--- ooni/api/spec.py | 33 +++++++++++++++++++++++++++++++-- ooni/nettest.py | 5 ++++- 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/data/ui/app/scripts/controllers.js b/data/ui/app/scripts/controllers.js index 5ba24e1..c2a9c0d 100644 --- a/data/ui/app/scripts/controllers.js +++ b/data/ui/app/scripts/controllers.js @@ -49,7 +49,17 @@ ooniprobe.controller('TestCtrl', ['$scope', '$routeParams', 'testStatus', 'Input ooniprobe.controller('TestBoxCtrl', ['$scope', 'startTest', function($scope, startTest) { + function hasAttributes(obj) { + var count = 0; + for (var i in obj) + count +=1; + if ( count == 0 ) { + return false; + } + return true; + } + $scope.manualFileInput = {}; $scope.startTest = function() { var options = {}; @@ -58,6 +68,13 @@ ooniprobe.controller('TestBoxCtrl', ['$scope', 'startTest', options[key] = option.value; }); + if (hasAttributes($scope.manualFileInput)) { + options['manual_input'] = {}; + angular.forEach($scope.manualFileInput, function(value, key) { + options['manual_input'][key] = value; + }); + } + startTest($scope.testDetails.id, options).success(function(){ $scope.updateTestStatus(); }); @@ -65,4 +82,15 @@ ooniprobe.controller('TestBoxCtrl', ['$scope', 'startTest', }]); +ooniprobe.controller('FileInput', ['$scope', + function($scope) { + + $scope.manualShow = false; + $scope.toggleManualInput = function() { + if ($scope.manualShow) + $scope.manualShow = false; + else + $scope.manualShow = true; + } +}]); diff --git a/data/ui/app/views/test.html b/data/ui/app/views/test.html index f92461a..fdd5931 100644 --- a/data/ui/app/views/test.html +++ b/data/ui/app/views/test.html @@ -8,11 +8,17 @@ <div ng-repeat="(name, options) in testDetails.arguments"> <div ng-switch on="options.type"> - <div ng-switch-when="file"> - <label>{{name}}</label> - <select ng-model="testDetails.arguments[name].value"> + <div ng-switch-when="file" ng-controller="FileInput"> + <label>{{name}} + <button class="btn btn-small" ng-click="toggleManualInput()">Toggle manual input</button> + </label> + <select ng-model="testDetails.arguments[name].value" ng-hide="manualShow"> <option ng-repeat="input in inputs" value="input.filename">{{input.filename}}</option> </select> + <br/> + <div class="manualFileInput" ng-show="manualShow"> + <textarea ng-model="manualFileInput[name]"></textarea> + </div> </div> <div ng-switch-default> diff --git a/ooni/api/spec.py b/ooni/api/spec.py index 071beaf..fe18ccc 100644 --- a/ooni/api/spec.py +++ b/ooni/api/spec.py @@ -3,6 +3,7 @@ import re import copy import json import types +import tempfile from twisted.python import usage from cyclone import web, escape @@ -122,6 +123,20 @@ def get_reporters(net_test_loader): reporters.append(oonib_reporter) return reporters +def write_temporary_input(content): + """ + Creates a temporary file for the given content. + + Returns: + the path to the temporary file. + """ + fd, path = tempfile.mkstemp() + with open(path, 'w') as f: + f.write(content) + f.close() + print "This is the path %s" % path + return fd, path + class StartTest(ORequestHandler): def post(self, test_name): """ @@ -129,11 +144,25 @@ class StartTest(ORequestHandler): """ test_file = oonidApplication.director.netTests[test_name]['path'] test_options = json.loads(self.request.body) + tmp_files = [] + if (test_options['manual_input']): + for option, content in test_options['manual_input'].items(): + fd, path = write_temporary_input(content) + test_options[option] = path + tmp_files.append((fd, path)) + test_options.pop('manual_input') + net_test_loader = get_net_test_loader(test_options, test_file) try: net_test_loader.checkOptions() - oonidApplication.director.startNetTest(net_test_loader, - get_reporters(net_test_loader)) + d = oonidApplication.director.startNetTest(net_test_loader, + get_reporters(net_test_loader)) + @d.addBoth + def cleanup(result): + for fd, path in tmp_files: + os.close(fd) + os.remove(path) + except MissingRequiredOption, option_name: self.write({'error': 'Missing required option: "%s"' % option_name}) diff --git a/ooni/nettest.py b/ooni/nettest.py index 66639f8..601fa24 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -132,8 +132,11 @@ def getArguments(test_class): for opt_parameter in test_class.usageOptions.optParameters: option_name = opt_parameter[0] + opt_type="text" + if opt_parameter[3].lower().startswith("file"): + opt_type="file" arguments[option_name] = getOption(opt_parameter, - test_class.requiredOptions) + test_class.requiredOptions, type=opt_type) return arguments

1 0

[ooni-probe/master] Merge branch 'master' into feature/ui
by art＠torproject.org 19 Jun '13

19 Jun '13

commit c8ee22a8afc3fd7f3525d1f1d4e717f3aadd2713 Merge: 673236c 5782f32 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun May 5 17:52:44 2013 +0200 Merge branch 'master' into feature/ui * master: (69 commits) Implement Vagrant file for running ooniprobe on any platform Have some slightly saner defaults for the config file Fix markup of README file Fix markup of code Swallow the report creation failure so that it does not propagate Fix tls_handshake input processing Fix import conflict Set self.input to self.host if no inputFile, and update several log messages. Reformat docstrings for several HandshakeTest functions to Sphinx format. Rename HandshakeTest.test_tlsv1_handshake() to HandshakeTest.test_handshake(). Cleanup docstrings for getX509Name() and getPublicKey() functions. Add Sphinx docstring for getPeerCert() function. Move to call IPAddress explicitly from ipaddr.IPAdress. Add self.host to the inputProcessor(). If no ciphersuite list is provided, explain that we're using firefox's. Fix bug where conflicting SSL context choices was not explained to the user. Fix #8440, set the self.host attribute when given the --host option. Create a host attribute for storing the --host option, and set self.ciphers. Make 'context' be a class-level attribute and give it a Sphinx :attr: docstring. Bump the version number on HandshakeTest, and rename the usageOptions attribute. ... Conflicts: data/ooniprobe.conf.sample ooni/director.py .gitignore | 2 + README.md | 161 +++++-- Vagrantfile | 153 ++++++ data/ooniprobe.conf.sample | 4 +- docs/fabfile.py | 22 + inputs/Makefile | 3 +- nettests/experimental/tls_handshake.py | 794 ++++++++++++++++++++++++++++++++ ooni/reporter.py | 3 +- ooni/utils/txscapy.py | 1 + setup.py | 4 +- 10 files changed, 1089 insertions(+), 58 deletions(-) diff --cc data/ooniprobe.conf.sample index 2b79d3e,0000000..a12b526 mode 100644,000000..100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@@ -1,55 -1,0 +1,55 @@@ +# This is the configuration file for OONIProbe +# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html +# Keep in mind that indentation matters. + +basic: + # Where OONIProbe should be writing it's log file + logfile: /var/log/ooniprobe.log +privacy: + # Should we include the IP address of the probe in the report? + includeip: false + # Should we include the ASN of the probe in the report? + includeasn: false + # Should we include the country as reported by GeoIP in the report? + includecountry: false + # Should we include the city as reported by GeoIP in the report? + includecity: false + # Should we collect a full packet capture on the client? + includepcap: false +reports: + # This is a packet capture file (.pcap) to load as a test: + pcap: null + collector: 'httpo://nkvphnp3p6agi5qq.onion' +advanced: + geoip_data_dir: /usr/share/ooni/ + debug: true + # enable if auto detection fails + #tor_binary: /usr/sbin/tor + # For auto detection + interface: auto + # Of specify a specific interface + #interface: wlan0 + # If you do not specify start_tor, you will have to have Tor running and + # explicitly set the control port and SOCKS port - start_tor: true ++ start_tor: false + # After how many seconds we should give up on a particular measurement + measurement_timeout: 30 + # After how many retries we should give up on a measurement + measurement_retries: 2 + # How many measurments to perform concurrently + measurement_concurrency: 100 + # After how may seconds we should give up reporting + reporting_timeout: 30 + # After how many retries to give up on reporting + reporting_retries: 3 + # How many reports to perform concurrently + reporting_concurrency: 20 +tor: - #socks_port: 9050 ++ socks_port: 9050 + #control_port: 9051 + # Specify the absolute path to the Tor bridges to use for testing + #bridges: bridges.list + # Specify path of the tor datadirectory. + # This should be set to something to avoid having Tor download each time + # the descriptors and consensus data. + #data_dir: ~/.tor/ diff --cc setup.py index 454984d,e336e08..1ae39d4 --- a/setup.py +++ b/setup.py @@@ -13,19 -11,9 +13,17 @@@ install_requires = ] dependency_links = [ - 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe', - 'https://github.com/hellais/pypcap/archive/v1.1.1.tar.gz#egg=pypcap-1.1.1' + 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe' ] +files = [] +for root, dirs, file_names in os.walk('data/'): + for file_name in file_names: + if not file_name.endswith('.pyc'): + files.append(os.path.join(root, file_name)) + +data_files = [('/usr/share/ooni/', files)] + with open('requirements.txt') as f: for line in f: if line.startswith("#") or line.startswith('http'):

1 0

[ooni-probe/master] Add cyclone as a dependency
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 28388e3522c3b82200f7afd2b748e2ac60525ead Author: Arturo Filastò <art(a)fuffa.org> Date: Tue May 7 18:59:27 2013 +0200 Add cyclone as a dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index bdf4cd0..4f951de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ zope.component>=4.0.0 zope.event>=4.0.0 zope.interface>=4.0.1 Pyrex +cyclone>=1.1 # # This is a Tor Project mirror with valid SSL/TLS certs that is stable and fast #

1 0

[ooni-probe/master] Implement XSRF protection in the HTTP API
by art＠torproject.org 19 Jun '13

19 Jun '13

commit ccf125bce7cd3094bfd4bb8b1ce6b8d2d6890e22 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue May 7 18:59:10 2013 +0200 Implement XSRF protection in the HTTP API --- ooni/api/spec.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ooni/api/spec.py b/ooni/api/spec.py index fe18ccc..f707253 100644 --- a/ooni/api/spec.py +++ b/ooni/api/spec.py @@ -4,6 +4,7 @@ import copy import json import types import tempfile +import functools from twisted.python import usage from cyclone import web, escape @@ -19,8 +20,18 @@ class InvalidInputFilename(Exception): class FilenameExists(Exception): pass +def check_xsrf(method): + @functools.wraps(method) + def wrapper(self, *args, **kw): + xsrf_header = self.request.headers.get("X-XSRF-TOKEN") + if self.xsrf_token != xsrf_header: + raise web.HTTPError(403, "Invalid XSRF token.") + return method(self, *args, **kw) + return wrapper + class ORequestHandler(web.RequestHandler): serialize_lists = True + xsrf_cookie_name = "XSRF-TOKEN" def write(self, chunk): """ @@ -33,6 +44,7 @@ class ORequestHandler(web.RequestHandler): web.RequestHandler.write(self, chunk) class Status(ORequestHandler): + @check_xsrf def get(self): result = {'active_tests': oonidApplication.director.activeNetTests} self.write(result) @@ -47,6 +59,8 @@ class Inputs(ORequestHandler): """ This handler is responsible for listing and adding new inputs. """ + + @check_xsrf def get(self): """ Obtain the list of currently installed inputs. Inputs are stored inside @@ -55,6 +69,7 @@ class Inputs(ORequestHandler): input_list = list_inputs() self.write(input_list) + @check_xsrf def post(self): """ Add a new input to the currently installed inputs. @@ -76,6 +91,8 @@ class Inputs(ORequestHandler): fp.write(body) class ListTests(ORequestHandler): + + @check_xsrf def get(self): test_list = copy.deepcopy(oonidApplication.director.netTests) for test_id in test_list.keys(): @@ -138,6 +155,8 @@ def write_temporary_input(content): return fd, path class StartTest(ORequestHandler): + + @check_xsrf def post(self, test_name): """ Starts a test with the specified options. @@ -174,6 +193,8 @@ class StartTest(ORequestHandler): 'Insufficient priviledges'}) class StopTest(ORequestHandler): + + @check_xsrf def delete(self, test_name): pass @@ -199,6 +220,8 @@ def get_test_results(test_id): return test_results class TestStatus(ORequestHandler): + + @check_xsrf def get(self, test_id): """ Returns the requested test_id details and the stored results for such

1 0

[ooni-probe/master] Bump to version 0.1.0
by art＠torproject.org 19 Jun '13

19 Jun '13

commit b73fa7c06a9fbd59ba1a792200e767e1b03ff669 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed May 8 16:44:02 2013 +0200 Bump to version 0.1.0 --- ooni/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 1810a7f..9663b08 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -7,7 +7,7 @@ from . import templates from . import utils __author__ = "Arturo Filastò" -__version__ = "0.0.12" +__version__ = "0.1.0" __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', diff --git a/setup.py b/setup.py index 1ae39d4..6ebca7b 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ with open('requirements.txt') as f: setup( name="ooni-probe", - version="0.0.12", + version="0.1.0", author="Arturo Filastò", author_email = "art(a)torproject.org", url="https://ooni.torproject.org/",

1 0

[ooni-probe/master] Include jquery as dep
by art＠torproject.org 19 Jun '13

19 Jun '13

commit ccc53b5cd5d7cf1e96b9595b0e5d380ca6dd69e9 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed May 8 19:10:58 2013 +0200 Include jquery as dep Fix bug in ng-upload --- data/ui/app/index.html | 1 + data/ui/app/libs/ng-upload/ng-upload.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data/ui/app/index.html b/data/ui/app/index.html index c304066..5f3e376 100644 --- a/data/ui/app/index.html +++ b/data/ui/app/index.html @@ -28,6 +28,7 @@ <script src="libs/angular/angular.js"></script> <script src="libs/angular-resource/angular-resource.js"></script> + <script src="libs/jquery/jquery.js"></script> <script src="libs/ng-upload/ng-upload.js"></script> <script src="scripts/app.js"></script> <script src="scripts/services.js"></script> diff --git a/data/ui/app/libs/ng-upload/ng-upload.js b/data/ui/app/libs/ng-upload/ng-upload.js index fae7a44..cedaa1f 100644 --- a/data/ui/app/libs/ng-upload/ng-upload.js +++ b/data/ui/app/libs/ng-upload/ng-upload.js @@ -38,7 +38,7 @@ angular.module('ngUpload', []) options.enableControls = attrs.uploadOptionsEnableControls; // submit the form - requires jQuery - var form = element.parents('form[ng-upload]') || element.parents('form.ng-upload'); + var form = $(element).parents('form[ng-upload]') || $(element).parents('form.ng-upload'); // Retrieve the callback function var fn = $parse(attrs.uploadSubmit); @@ -104,4 +104,4 @@ angular.module('ngUpload', []) element.attr("encoding", "multipart/form-data"); } }; - }]); \ No newline at end of file + }]);

1 0

[ooni-probe/master] Fix bug in HTTP api
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 21ae4fb974e3eea55929e1613d29d1675fe4e03a Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 17:45:19 2013 +0200 Fix bug in HTTP api --- ooni/api/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/api/spec.py b/ooni/api/spec.py index f707253..ecd514c 100644 --- a/ooni/api/spec.py +++ b/ooni/api/spec.py @@ -164,7 +164,7 @@ class StartTest(ORequestHandler): test_file = oonidApplication.director.netTests[test_name]['path'] test_options = json.loads(self.request.body) tmp_files = [] - if (test_options['manual_input']): + if ('manual_input' in test_options): for option, content in test_options['manual_input'].items(): fd, path = write_temporary_input(content) test_options[option] = path

1 0

[ooni-probe/master] Add oonid script to run ooni daemon
by art＠torproject.org 19 Jun '13

19 Jun '13

commit 22c0c16f8eb9b7fa5680e8f23167ded4a7082835 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jun 3 17:56:11 2013 +0200 Add oonid script to run ooni daemon --- bin/oonid | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/oonid b/bin/oonid new file mode 100755 index 0000000..dd59eb9 --- /dev/null +++ b/bin/oonid @@ -0,0 +1,7 @@ +#!/bin/bash +# XXX This is very Ghetto. The proper way to do this is to make a twisted +# plugin. The plugin will then be installed and can then be run directly with +# `twistd oonid` + +OONID_PATH=`python -c 'import ooni;import os;print os.path.join(os.path.dirname(os.path.abspath(ooni.__file__)), "oonid.py")'` +twistd -y $OONID_PATH

1 0