May 2012 - tor-commits - lists.torproject.org

[tor/maint-0.2.2] Fix more clang format-nonliteral warnings (bug 5969)
by nickm＠torproject.org 31 May '12

31 May '12

commit 3a9351b57e528b1d0bd2e72bcf78db7c91b2ff8f Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 30 19:57:02 2012 -0400 Fix more clang format-nonliteral warnings (bug 5969) --- src/common/log.c | 7 +++++++ src/or/control.c | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/common/log.c b/src/common/log.c index ac98f13..f2999f4 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -137,6 +137,13 @@ static void close_log(logfile_t *victim); static char *domain_to_string(log_domain_mask_t domain, char *buf, size_t buflen); +static INLINE char *format_msg(char *buf, size_t buf_len, + log_domain_mask_t domain, int severity, const char *funcname, + const char *format, va_list ap, size_t *msg_len_out) + CHECK_PRINTF(6,0); +static void logv(int severity, log_domain_mask_t domain, const char *funcname, + const char *format, va_list ap) + CHECK_PRINTF(4,0); /** Name of the application: used to generate the message we write at the * start of each new log. */ diff --git a/src/or/control.c b/src/or/control.c index d6e6932..8aa4240 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -126,6 +126,13 @@ typedef int event_format_t; static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); +static void send_control_event_impl(uint16_t event, event_format_t which, + const char *format, va_list ap) + CHECK_PRINTF(3,0); +static int control_event_status(int type, int severity, const char *format, + va_list args) + CHECK_PRINTF(3,0); + static void send_control_done(control_connection_t *conn); static void send_control_event(uint16_t event, event_format_t which, const char *format, ...) @@ -3918,6 +3925,7 @@ control_event_my_descriptor_changed(void) static int control_event_status(int type, int severity, const char *format, va_list args) { + char *user_buf = NULL; char format_buf[160]; const char *status, *sev; @@ -3949,13 +3957,15 @@ control_event_status(int type, int severity, const char *format, va_list args) log_warn(LD_BUG, "Unrecognized status severity %d", severity); return -1; } - if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s %s\r\n", - status, sev, format)<0) { + if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s\r\n", + status, sev)<0) { log_warn(LD_BUG, "Format string too long."); return -1; } + tor_vasprintf(&user_buf, format, args); - send_control_event_impl(type, ALL_FORMATS, format_buf, args); + send_control_event(type, ALL_FORMATS, "%s %s", format_buf, user_buf); + tor_free(user_buf); return 0; }

1 0

[tor/maint-0.2.2] Add __attribute__(format)s for our varargs printf/scanf wrappers
by nickm＠torproject.org 31 May '12

31 May '12

commit e28489467233bff4500a70f8a7b22e42ca3b3e68 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 30 12:14:38 2012 -0400 Add __attribute__(format)s for our varargs printf/scanf wrappers It turns out that if you set the third argument of __attribute__(format) to 0, GCC and Clang will check the format argument without expecting to find variadic arguments. This is the correct behavior for vsnprintf, vasprintf, and vscanf. I'm hoping this will fix bug 5969 (a clang warning) by telling clang that the format argument to tor_vasprintf is indeed a format string. --- changes/bug5969_022 | 7 +++++++ src/common/compat.h | 5 +++-- src/common/util.h | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/changes/bug5969_022 b/changes/bug5969_022 new file mode 100644 index 0000000..57c8744 --- /dev/null +++ b/changes/bug5969_022 @@ -0,0 +1,7 @@ + o Minor bugfixes + - Fix a build warning with Clang 3.1 related to our use of vasprint. + Fix for bug 5969. Bugfix on 0.2.2.11-alpha. + + o Compilation improvements: + - Tell GCC and Clang to check for any errors in format strings passed + to the tor_v*(print|scan)f functions. diff --git a/src/common/compat.h b/src/common/compat.h index d2f1fd1..fc70caf 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -261,11 +261,12 @@ void tor_munmap_file(tor_mmap_t *handle) ATTR_NONNULL((1)); int tor_snprintf(char *str, size_t size, const char *format, ...) CHECK_PRINTF(3,4) ATTR_NONNULL((1,3)); int tor_vsnprintf(char *str, size_t size, const char *format, va_list args) - ATTR_NONNULL((1,3)); + CHECK_PRINTF(3,0) ATTR_NONNULL((1,3)); int tor_asprintf(char **strp, const char *fmt, ...) CHECK_PRINTF(2,3); -int tor_vasprintf(char **strp, const char *fmt, va_list args); +int tor_vasprintf(char **strp, const char *fmt, va_list args) + CHECK_PRINTF(2,0); const void *tor_memmem(const void *haystack, size_t hlen, const void *needle, size_t nlen) ATTR_PURE ATTR_NONNULL((1,3)); diff --git a/src/common/util.h b/src/common/util.h index b9db25c..d477156 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -211,7 +211,11 @@ const char *escaped(const char *string); struct smartlist_t; void wrap_string(struct smartlist_t *out, const char *string, size_t width, const char *prefix0, const char *prefixRest); -int tor_vsscanf(const char *buf, const char *pattern, va_list ap); +int tor_vsscanf(const char *buf, const char *pattern, va_list ap) +#ifdef __GNUC__ + __attribute__((format(scanf, 2, 0))) +#endif + ; int tor_sscanf(const char *buf, const char *pattern, ...) #ifdef __GNUC__ __attribute__((format(scanf, 2, 3)))

1 0

[ooni-probe/master] Update HACKING file to reflect directory structure changes
by art＠torproject.org 31 May '12

31 May '12

commit 5eddf83fa5d9bc97f439bc0be7152aed1a204a55 Author: Arturo Filastò <hellais(a)torproject.org> Date: Tue May 29 01:35:22 2012 +0200 Update HACKING file to reflect directory structure changes --- HACKING | 36 +++++++++++++++++++++++------------- 1 files changed, 23 insertions(+), 13 deletions(-) diff --git a/HACKING b/HACKING index ed8d5dc..821d524 100644 --- a/HACKING +++ b/HACKING @@ -36,45 +36,55 @@ Code Structure - HACKING The document you are currently reading. -- assets/ +- oonib/ + Contains the OONI probe backend to be run on the ooni-net + +- oonid/ + Contains the OONI daemon that can be used to interrogated from the cli to + run tests. + +- ooni/ + Contains the main ooni probe comand line client + +- ooni/assets/ Where we store all the asset files that are used when running OONI tests. -- config.py +- ooni/config.py Parts of the code related to parsing OONI configuration files and making them accessible to other components of the software. -- logo.py +- ooni/logo.py File containing some funny ASCII art. Yes, we do enjoy ASCII art and are not afraid to admit it! -- nodes.conf +- ooni/nodes.conf The configuration file for nodes. This contains the list of network and code execution nodes that can be used to run tests off of. -- ooniprobe.py +- ooni/ooniprobe.py The main OONI-probe command line interface. This is responsible for parsing the command line arguments and passing the arguments to the underlying components. -- ooni-probe.conf +- ooni/ooni-probe.conf The main OONI-probe configuration file. This can be used to configure your OONI CLI, tell it where it should report to, where the asset files are located, what should be used for control, etc. -- plugoo/__init__.py +- ooni/plugoo/__init__.py All the necessary "goo" for making OONI probe work. This means loading Assets, creating Reports, running Tests, interacting with Nodes. -- plugoo/assets.py +- ooni/plugoo/assets.py This is a python object representation of the data that is located inside the asset directory. -- plugoo/nodes.py +- ooni/plugoo/nodes.py The part of code responsible for interacting with OONI Nodes. Nodes can be Network or Code Execution. Network nodes are capable of receiving packets and fowarding them onto their @@ -83,19 +93,19 @@ Code Structure of work, they are therefore capable of receiving a set of tests that should be completed by a set of Network nodes or run locally. -- plugoo/reports.py +- ooni/plugoo/reports.py Takes care of transforming the output of a test into a report. This may mean pushing the result data to a remote backend or simply writing a local file. -- plugoo/tests.py +- ooni/plugoo/tests.py The main OONI probe test class. This provides all the necessary scaffold to write your own test based on the OONI paradigm. -- tests/ +- ooni/oonitests/ Contains all the "offical" OONI tests that are shipped. -- utils.py +- ooni/utils.py Helper functions that don't fit into any place, but are not big enough to be a dependency by themselves.

1 0

[ooni-probe/master] Get the new OONI architecture working properly.
by art＠torproject.org 31 May '12

31 May '12

commit c4c91d393ba3956b84d46521e2f58d30584b3ea0 Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 03:31:05 2012 +0200 Get the new OONI architecture working properly. (kudos to the organizers of FreeBird :) ) --- .gitignore | 3 +- ooni/oonicli.py | 70 ++++++++---------------- ooni/ooniprobe.log | 3 +- ooni/plugins/.bridget.py.swp | Bin 12288 -> 0 bytes ooni/plugins/.dnstamper.py.swp | Bin 12288 -> 0 bytes ooni/plugins/blocking.py | 41 ++++++++++++++ ooni/plugins/bridget.py | 36 ------------ ooni/plugins/dnstamper.py | 22 -------- ooni/plugins/dropin.cache | 90 ++++++++++++++++++++++++++++++- ooni/plugins/skel.py | 9 +++- ooni/plugoo/tests.py | 118 ++++++++++++++-------------------------- ooni/plugoo/work.py | 64 +++++++++++++++------- ooni/skel.py | 10 ---- oonib/oonibackend.py | 8 ++-- 14 files changed, 255 insertions(+), 219 deletions(-) diff --git a/.gitignore b/.gitignore index 6433d66..8ed47e9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ proxy-lists/ip-cc.txt proxy-lists/ips.txt proxy-lists/italy-dns-ips.txt proxy-lists/italy-http-ips.txt -private/* \ No newline at end of file +private/* +ooni/plugins/dropin.cache diff --git a/ooni/oonicli.py b/ooni/oonicli.py index d290e68..a0272fb 100755 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -9,7 +9,6 @@ # import sys -from plugoo import tests, work, assets from twisted.python import usage from twisted.plugin import getPlugins @@ -19,8 +18,10 @@ from zope.interface.exceptions import BrokenImplementation from zope.interface.exceptions import BrokenMethodImplementation from zope.interface.verify import verifyObject from pprint import pprint -from logo import getlogo -import plugins + +from ooni.plugoo import tests, work, assets +from ooni.logo import getlogo +from ooni import plugins __version__ = "0.0.1-prealpha" @@ -46,33 +47,20 @@ def retrieve_plugoo(): plugoo = retrieve_plugoo() -class StupidAsset(object): - def __init__(self): - self.idx = 0 - - def __iter__(self): - return self - - def next(self): - if self.idx > 30: - raise StopIteration - self.idx += 1 - return self.idx - - def runTest(test, options, global_options): - asset = None - if options['asset']: - print options['asset'] - asset = assets.Asset(options['asset']) - print asset - wgen = work.WorkGenerator(asset, plugoo[test].__class__, - dict(options), start=options['resume']) + parallelism = int(global_options['parallelism']) + worker = work.Worker(parallelism) + test = plugoo[test].__class__ + + #if options['asset']: + # print options['asset'] + # asset = assets.Asset(options['asset']) + # print asset - if global_options['parallelism']: - wgen.size = int(global_options['parallelism']) - worker = work.Worker(wgen.size) + wgen = work.WorkGenerator(test(options, global_options), + dict(options), + start=options['resume']) for x in wgen: worker.push(x) @@ -86,16 +74,17 @@ class Options(usage.Options): subCommands.append([test, None, plugoo[test].options, "Run the %s test" % test]) optFlags = [ - ['local', 'l', "If the test should be run locally (without having oonid running)"], - ['status', 'x', 'Show current state'], - ['restart', 'r', 'Restart OONI'] + #['remote', 'r', "If the test should be run remotely (not supported)"], + #['status', 'x', 'Show current state'], + #['restart', 'r', 'Restart OONI'] ] optParameters = [ ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], - ['target-node', 't', 'localhost:31415', 'Select target node'], - ['ooninet', 'o', 'localhost:4242', "Select OONI-net address for reporting"], - ['password', 'p', 'opennetwork', "Specify the password for authentication"], + #['target-node', 't', 'localhost:31415', 'Select target node'], + ['output', 'o', 'report.log', "Specify output report file"], + ['log', 'l', 'oonicli.log', "Specify output log file"], + #['password', 'p', 'opennetwork', "Specify the password for authentication"], ] def opt_version(self): @@ -116,24 +105,11 @@ class Options(usage.Options): config = Options() config.parseOptions() -if config['status']: - print "oonid is not running." - sys.exit(0) - -if config['restart']: - print "Restarting oonid." - sys.exit(0) - if not config.subCommand: print "Error! No Test Specified." config.opt_help() sys.exit(1) -if config['local']: - runTest(config.subCommand, config.subOptions, config) +runTest(config.subCommand, config.subOptions, config) -else: - print "This feature is currently not supported. :(" - print "Use -l to run the test locally." - sys.exit(0) diff --git a/ooni/ooniprobe.log b/ooni/ooniprobe.log index d3914dc..700e820 100644 --- a/ooni/ooniprobe.log +++ b/ooni/ooniprobe.log @@ -1,2 +1 @@ -2012-05-29 01:32:22,811 ooniprobe INFO Started ooni-probe -2012-05-29 01:32:22,972 ooniprobe WARNING Soft fail 'module' object has no attribute '__plugoo__' +2012-05-31 02:27:28,010 ooniprobe INFO Started ooni-probe diff --git a/ooni/plugins/.bridget.py.swp b/ooni/plugins/.bridget.py.swp deleted file mode 100644 index 226d826..0000000 Binary files a/ooni/plugins/.bridget.py.swp and /dev/null differ diff --git a/ooni/plugins/.dnstamper.py.swp b/ooni/plugins/.dnstamper.py.swp deleted file mode 100644 index 9acb1c9..0000000 Binary files a/ooni/plugins/.dnstamper.py.swp and /dev/null differ diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py new file mode 100644 index 0000000..bbcb1a2 --- /dev/null +++ b/ooni/plugins/blocking.py @@ -0,0 +1,41 @@ +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, TwistedTest + +class BlockingArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['other', 'o', None, 'Other arguments']] + +class BlockingTest(TwistedTest): + implements(IPlugin, ITest) + + shortName = "blocking" + description = "Blocking plugin" + requirements = None + options = BlockingArgs + # Tells this to be blocking. + blocking = True + + def control(self, experiment_result, args): + print "Experiment Result:", experiment_result + print "Args", args + return experiment_result + + def experiment(self, args): + import urllib + req = urllib.urlopen(args['asset']) + return {'page': req.readlines()} + + def load_assets(self): + if self.local_options: + return {'asset': Asset(self.local_options['asset'])} + else: + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +blocking = BlockingTest(None, None, None) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py deleted file mode 100644 index 7271468..0000000 --- a/ooni/plugins/bridget.py +++ /dev/null @@ -1,36 +0,0 @@ -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from plugoo.tests import ITest, TwistedTest -from twisted.internet import threads - -from tests.bridget import BridgeT as BridgeTlegacy -from tests.bridget import BridgeTAsset as BridgeTAsset -from ooniprobe import ooni - -o = ooni() - -class BridgetArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index'], - ['bridge', 'b', None, 'Specify a single bridge']] - -class BridgeT(TwistedTest): - implements(IPlugin, ITest) - - shortName = "bridget" - description = "Bridget plugin" - requirements = None - options = BridgetArgs - - def experiment(self): - bridget = BridgeTlegacy(o) - o.logger.info("Starting bridget test") - print "ASSET:%s " % self.asset - d = threads.deferToThread(bridget.connect, self.asset) - d.addCallback(self.d_experiment.callback, None) - return d - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -bridget = BridgeT(None, None) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py deleted file mode 100644 index b8b7c7f..0000000 --- a/ooni/plugins/dnstamper.py +++ /dev/null @@ -1,22 +0,0 @@ -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from plugoo.tests import ITest, TwistedTest - -class SkelArgs(usage.Options): - optParameters = [['hosts', 'f', None, 'Hostname asset file'], - ['dnsservers', 'd', None, 'DNS server asset file'], - ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] - -class SkelTest(TwistedTest): - implements(IPlugin, ITest) - - shortName = "skeleton" - description = "Skeleton plugin" - requirements = None - options = SkelArgs - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -skel = SkelTest(None, None) diff --git a/ooni/plugins/dropin.cache b/ooni/plugins/dropin.cache index 9258b18..bd26c09 100755 --- a/ooni/plugins/dropin.cache +++ b/ooni/plugins/dropin.cache @@ -53,7 +53,7 @@ g5 NtRp24 (dp25 g8 -S'plugins.bridget' +S'ooni.plugins.bridget' p26 sg10 Nsg11 @@ -73,4 +73,92 @@ sg21 S'bridget' p31 sg10 +NsbasbsS'blocking' +p32 +g3 +(g4 +g5 +NtRp33 +(dp34 +g8 +S'ooni.plugins.blocking' +p35 +sg10 +Nsg11 +(lp36 +g3 +(g13 +g5 +NtRp37 +(dp38 +g16 +(lp39 +g18 +acooni.plugoo.tests +ITest +p40 +asg20 +g33 +sg21 +S'blocking' +p41 +sg10 +NsbasbsS'httphost' +p42 +g3 +(g4 +g5 +NtRp43 +(dp44 +g8 +S'ooni.plugins.httphost' +p45 +sg10 +S'\n HTTP Host based filtering\n *************************\n\n This test detect HTTP Host field\n based filtering.\n' +p46 +sg11 +(lp47 +g3 +(g13 +g5 +NtRp48 +(dp49 +g16 +(lp50 +g18 +ag40 +asg20 +g43 +sg21 +S'httphost' +p51 +sg10 +NsbasbsS'tcpscan' +p52 +g3 +(g4 +g5 +NtRp53 +(dp54 +g8 +S'plugins.tcpscan' +p55 +sg10 +Nsg11 +(lp56 +g3 +(g13 +g5 +NtRp57 +(dp58 +g16 +(lp59 +g18 +ag40 +asg20 +g53 +sg21 +S'tcpscan' +p60 +sg10 Nsbasbs. \ No newline at end of file diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py index 93bc1bb..00e4311 100644 --- a/ooni/plugins/skel.py +++ b/ooni/plugins/skel.py @@ -15,7 +15,14 @@ class SkelTest(TwistedTest): description = "Skeleton plugin" requirements = None options = SkelArgs + blocking = False + + def load_assets(self): + if self.local_options: + return {'asset': open(self.local_options['asset'])} + else: + return {} # We need to instantiate it otherwise getPlugins does not detect it # XXX Find a way to load plugins without instantiating them. -skel = SkelTest(None, None) +skel = SkelTest(None, None, None) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 09363a2..75d0765 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -6,9 +6,13 @@ from zope.interface import Interface, Attribute import logging import itertools import gevent -from twisted.internet import reactor, defer + +from twisted.internet import reactor, defer, threads from twisted.python import failure -from plugoo.reports import Report + +from ooni.plugoo import assets, work +from ooni.plugoo.reports import Report + class Test: """ @@ -128,102 +132,64 @@ class ITest(Interface): #node = Attribute("""This represents the node that will run the test""") options = Attribute("""These are the arguments to be passed to the test for it's execution""") - def startTest(): + blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""") + + def startTest(asset): """ Launches the Test with the specified arguments on a node. """ -#class HTTPRequestTest(HTTPClient): -class HTTPRequestTest(object): - """ - This is an example of how I would like to be able to write a test. - - *BEWARE* this actually does not currently work, it's just an example of the - kind of API that I am attempting to achieve to simplify the writing of - tests. +class TwistedTest(object): + blocking = False - implements(ITest) + def __init__(self, local_options, global_options, ooninet=None): + self.local_options = local_options + self.global_options = global_options + self.assets = self.load_assets() + #self.ooninet = ooninet - """ - def startTest(): - # The response object should also contain the request + def load_assets(self): """ - response = {'response': {'headers': ..., 'content': ..., - 'runtime': ..., 'timestamp': ...}, - 'request': {'headers': ..., 'content', 'timestamp', ...} - } - response = self.http_request(address, headers) - if response.headers['content'].matches("Some string"): - self.censorship = True - return response - else: - self.censorship = False - return response - + This method should be overriden by the test writer to provide the logic + for loading their assets. """ - pass - -class TwistedTest(object): - def __init__(self, asset, arguments, ooninet=None): - self.asset = asset - self.arguments = arguments - self.start_time = datetime.now() - self._parse_arguments() - #self.ooninet = ooninet + return {'asset': None} def __repr__(self): - return "<TwistedTest %s %s>" % (self.arguments, self.asset) + return "<TwistedTest %s %s %s>" % (self.options, self.global_options, + self.assets) - def _parse_arguments(self): - print self.arguments - if self.arguments and 'test' in self.arguments: - self.test = self.arguments['test'] - - def finished(self, result): + def finished(self, control): #self.ooninet.report(result) - print "FINIHSED" self.end_time = datetime.now() + result = {} result['start_time'] = self.start_time result['end_time'] = self.end_time result['run_time'] = self.end_time - self.start_time + result['control'] = control + print "FINISHED", result + return result - def _do_experiment(self): - self.d = defer.maybeDeferred(self.experiment) - self.d.addCallback(self.control) + def _do_experiment(self, args): + if self.blocking: + self.d = threads.deferToThread(self.experiment, args) + else: + self.d = defer.maybeDeferred(self.experiment, args) + + self.d.addCallback(self.control, args) self.d.addCallback(self.finished) return self.d - def control(self, exp): + def control(self, result, args): print "Doing control..." - self.d.callback(result) + return result - def experiment(self): + def experiment(self, args): print "Doing experiment" - self.d_experiment.callback(None) - - def startTest(self): - print "Starting test %s" % repr(self) - return self._do_experiment() - -class TwistedTestFactory(object): + return {} - test = None - - def __init__(self, assets, node, - idx=0): - """ - """ - self.assets = assets - self.node = node - self.idx = idx - self.workunit = WorkUnitFactory(assets) - - def build_test(self): - """ - Returns a TwistedTest instance - """ - workunit = self.workunit.next() - t = self.test(node, workunit) - t.factory = self - return t + def startTest(self, args): + self.start_time = datetime.now() + print "Starting test %s" % self.__class__ + return self._do_experiment(args) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 7be4758..55bbd1d 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -10,15 +10,16 @@ :license: see LICENSE for more details. """ -from datetime import datetime +import itertools import yaml +from datetime import datetime from zope.interface import Interface, Attribute from twisted.python import failure from twisted.internet import reactor, defer -from plugoo.nodes import LocalNode +from ooni.plugoo.nodes import LocalNode class Worker(object): """ @@ -34,9 +35,9 @@ class Worker(object): self._running -= 1 if self._running < self.maxconcurrent and self._queued: workunit, d = self._queued.pop(0) - for work in workunit: - self._running += 1 - actuald = work.startTest().addBoth(self._run) + asset, test, idx = workunit + self._running += 1 + actuald = test.startTest(asset).addBoth(self._run) if isinstance(r, failure.Failure): r.trap() @@ -44,14 +45,13 @@ class Worker(object): print r['start_time'] print r['end_time'] print r['run_time'] - print repr(r) return r def push(self, workunit): if self._running < self.maxconcurrent: - for work in workunit: - self._running += 1 - work.startTest().addBoth(self._run) + asset, test, idx = workunit + self._running += 1 + test.startTest(asset).addBoth(self._run) return d = defer.Deferred() self._queued.append((workunit, d)) @@ -59,6 +59,8 @@ class Worker(object): class WorkUnit(object): """ + XXX This is currently not implemented for KISS sake. + This is an object responsible for completing WorkUnits it will return its result in a deferred. @@ -77,21 +79,21 @@ class WorkUnit(object): test = None arguments = None - def __init__(self, asset, test, idx, arguments=None): + def __init__(self, asset, assetNames, test, idx): self.asset = asset if not asset: self.assetGenerator = iter([1]) else: self.assetGenerator = iter(asset) self.Test = test - self.arguments = arguments + self.assetNames = assetNames self.idx = idx def __iter__(self): return self def __repr__(self): - return "<WorkUnit %s %s %s>" % (self.arguments, self.Test, self.idx) + return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx) def serialize(self): """ @@ -105,7 +107,7 @@ class WorkUnit(object): """ try: asset = self.assetGenerator.next() - ret = self.Test(asset, self.arguments) + ret = self.Test.set_asset(asset) return ret except StopIteration: raise StopIteration @@ -120,10 +122,12 @@ class WorkGenerator(object): """ size = 10 - def __init__(self, asset, test, arguments=None, start=None): - self.assetGenerator = asset + def __init__(self, test, arguments=None, start=None): self.Test = test - self.arguments = arguments + + self.assetGenerator = itertools.product(*self.Test.assets.values()) + self.assetNames = self.Test.assets.keys() + self.idx = 0 self.end = False if start: @@ -139,12 +143,32 @@ class WorkGenerator(object): self.idx += 1 def next(self): + if not self.assetGenerator: + self.end = True + return (self.assetNames, self.Test, self.idx) + + try: + asset = self.assetGenerator.next() + ret = {} + for i, v in enumerate(asset): + ret[self.assetNames[i]] = v + except StopIteration: + raise StopIteration + + self.idx += 1 + print "IDX:%s" % self.idx + return (ret, self.Test, self.idx) + + def p_next(self): + """ + XXX This is not used for KISS sake. + """ if self.end: raise StopIteration if not self.assetGenerator: self.end = True - return WorkUnit(None, self.Test, self.idx, self.arguments) + return WorkUnit(None, self.assetNames, self.Test, self.idx) # Plank asset p_asset = [] @@ -152,11 +176,13 @@ class WorkGenerator(object): try: asset = self.assetGenerator.next() p_asset.append(asset) - print asset + print p_asset except StopIteration: + if self.asset_num > 1: + pass self.end = True break self.idx += 1 - return WorkUnit(p_asset, self.Test, self.idx, self.arguments) + return WorkUnit(p_asset, self.assetNames, self.Test, self.idx) diff --git a/ooni/skel.py b/ooni/skel.py deleted file mode 100644 index 8982106..0000000 --- a/ooni/skel.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -""" - XXX - *** - - :copyright: (c) 2012 by Arturo Filastò - :license: see LICENSE for more details. -""" - diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 774a9c5..fc5bbdb 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -16,19 +16,19 @@ from twisted.web import resource, server, static from twisted.web.microdom import escape from twisted.names import dns -from backend.httpbackend import HTTPBackend -from backend.dnsbackend import ProxyDNSServer +from oonib.httpbackend import HTTPBackend +from oonib.dnsbackend import ProxyDNSServer # This tells twisted to set the server.version = "Apache" application = service.Application('oonibackend') serviceCollection = service.IServiceCollection(application) -internet.TCPServer(8000, server.Site(HTTPBackend())).setServiceParent(serviceCollection) +internet.TCPServer(2000, server.Site(HTTPBackend())).setServiceParent(serviceCollection) # Start the DNS Server related services TCPDNSServer = ProxyDNSServer() internet.TCPServer(8002, TCPDNSServer).setServiceParent(serviceCollection) UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer) -internet.UDPServer(5353, UDPFactory).setServiceParent(serviceCollection) +internet.UDPServer(5354, UDPFactory).setServiceParent(serviceCollection)

1 0

[ooni-probe/master] Prove how easy it is to port old test to the new model
by art＠torproject.org 31 May '12

31 May '12

commit 5028911580a1c003f4dbf1dff0bd8043c528ef72 Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 03:32:04 2012 +0200 Prove how easy it is to port old test to the new model --- ooni/plugins/httphost.py | 121 ++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 121 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py new file mode 100644 index 0000000..17e2e11 --- /dev/null +++ b/ooni/plugins/httphost.py @@ -0,0 +1,121 @@ +""" + HTTP Host based filtering + ************************* + + This test detect HTTP Host field + based filtering. +""" +import os +from datetime import datetime + +import urllib2 +import httplib + +from BeautifulSoup import BeautifulSoup + +# XXX reduce boilerplating. +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, TwistedTest + +class HTTPHostArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['controlserver', 'c', None, 'Specify the control server'], + ['resume', 'r', 0, 'Resume at this index'], + ['other', 'o', None, 'Other arguments']] + def control(self, experiment_result, args): + print "Experiment Result:", experiment_result + print "Args", args + return experiment_result + + def experiment(self, args): + import urllib + req = urllib.urlopen(args['asset']) + return {'page': req.readlines()} + +class HTTPHostAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + return line.split(',')[1].replace('\n','') + + +class HTTPHostTest(TwistedTest): + implements(IPlugin, ITest) + + shortName = "httphost" + description = "HTTP Host plugin" + requirements = None + options = HTTPHostArgs + # Tells this to be blocking. + blocking = True + + def check_response(self, response): + soup = BeautifulSoup(response) + if soup.head.title.string == "WikiLeaks": + # Response indicates censorship + return True + else: + # Response does not indicate censorship + return False + + def is_censored(self, response): + if response: + soup = BeautifulSoup(response) + censored = self.check_response(response) + else: + censored = "unreachable" + return censored + + def urllib2_test(self, control_server, host): + req = urllib2.Request(control_server) + req.add_header('Host', host) + try: + r = urllib2.urlopen(req) + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + def httplib_test(self, control_server, host): + try: + conn = httplib.HTTPConnection(control_server) + conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) + conn.putheader("Host", host) + conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") + conn.endheaders() + r = conn.getresponse() + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + respopnse = None + + return (censored, response) + + def experiment(self, args): + control_server = self.local_options['controlserver'] + print args + censored = self.httplib_test(control_server, args['asset']) + return {'control': control_server, + 'host': args['asset'], + 'censored': censored} + + def load_assets(self): + if self.local_options: + return {'asset': Asset(self.local_options['asset'])} + else: + return {} + +httphost = HTTPHostTest(None, None, None)

1 0

[ooni-probe/master] Merge branch 'master' into mastermerge
by art＠torproject.org 31 May '12

31 May '12

commit c7145bb3177eefcfcebb013db379f34b4c8baf35 Merge: a319db3 abfcf7c Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 04:44:39 2012 +0200 Merge branch 'master' into mastermerge Conflicts: ooni/plugoo/reports.py HACKING | 2 +- LICENSE | 26 +++++++++ TODO | 5 ++ ooni/ooni-probe.conf | 7 ++- ooni/oonitests/dnstamper.py | 122 ++++++++++++++++++++++++++++++++++++------ ooni/oonitests/httphost.py | 5 ++- 6 files changed, 146 insertions(+), 21 deletions(-) diff --cc ooni/ooni-probe.conf index 336af99,0000000..d95e410 mode 100644,000000..100644 --- a/ooni/ooni-probe.conf +++ b/ooni/ooni-probe.conf @@@ -1,72 -1,0 +1,77 @@@ +# ooni-probe +# +# These are the global configuration parameters necessary to +# make ooni-probe work +[main] +reportdir = reports/ +logfile = ooniprobe.log +assetdir = assets/ +testdir = oonitests/ + +loglevel = DEBUG +consoleloglevel = DEBUG +proxyaddress = 127.0.0.1:9050 + +# The following configurations are for searching for PlanetLab +# nodes, adding them to a slice, and PlanetLab general API +# authentication: +pl_username = yourusername +pl_password = yourpassword + +# These are configurations specific to the tests that should be +# run by ooni-probe +[tests] +run = dnstamper +### DNS testing related config parameters + +# This is the list of hostnames that must be looked up +dns_experiment = top-1m.txt + +# This is the dns servers to be tested +dns_experiment_dns = dns_servers.txt + +# This is the control known good DNS server - dns_control_server = 8.8.8.8 ++dns_control_server = 91.191.136.152 ++ ++# Specify whether the dnstamper test should attempt to remove ++# GeoIP-based false positives by doing a reverse DNS resolve ++# on positive results. ++dns_reverse_lookup = true + +### traceroute testing related config parameters + +# This is the list of ips to traceroute to +traceroute = example_exp_list.txt + +# This is the list of ports that should be used +# src_x,src_y,src_z|dst_x,dst_y,dst_z +traceroute_ports = 0,53,80,123,443|0,53,80,123,443 + +# The protocol to be used in the scan +traceroute_proto = UDP, TCP, ICMP + +### keyword injection related tests + +# List of keywords +keywords = keywordlist.txt + +# hosts +keywords_hosts = hostslist.txt + +# Methods to be used for testing +keyword_method = http,telnet + +### Tor bridge testing + +tor_bridges = bridgetests.txt +tor_bridges_timeout = 40 + +[report] +file = report.log +timestamp = true +#ssh = 127.0.0.1:22 +#ssh_user = theusername +#ssh_password = thepassword +#ssh_keyfile = ~/.ssh/mykey_rsa +#ssh_rpath = ~/ooni-probe/ +#tcp = "127.0.0.1:9088" diff --cc ooni/oonitests/dnstamper.py index 68be12d,0000000..498ba04 mode 100644,000000..100644 --- a/ooni/oonitests/dnstamper.py +++ b/ooni/oonitests/dnstamper.py @@@ -1,70 -1,0 +1,156 @@@ ++# -*- coding: utf-8 -*- ++""" ++ dnstamper ++ ********* ++ ++ This test resolves DNS for a list of domain names, one per line, in the ++ file specified in the ooni-config under the setting "dns_experiment". If ++ the file is top-1m.txt, the test will be run using Amazon's list of top ++ one million domains. The experimental dns servers to query should ++ be specified one per line in assets/dns_servers.txt. ++ ++ The test reports censorship if the cardinality of the intersection of ++ the query result set from the control server and the query result set ++ from the experimental server is zero, which is to say, if the two sets ++ have no matching results whatsoever. ++ ++ NOTE: This test frequently results in false positives due to GeoIP-based ++ load balancing on major global sites such as google, facebook, and ++ youtube, etc. ++ ++ :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft ++ :license: see LICENSE for more details ++""" ++ ++try: ++ from dns import resolver, reversename ++except: ++ print "Error: dnspython is not installed! (http://www.dnspython.org/)" +try: - from dns import resolver ++ import gevent +except: - print "Error dnspython is not installed! (http://www.dnspython.org/)" - import gevent ++ print "Error: gevent is not installed! (http://www.gevent.org/)" ++ +import os ++ +import plugoo +from plugoo.assets import Asset +from plugoo.tests import Test + - +__plugoo__ = "DNST" +__desc__ = "DNS censorship detection test" + ++class Top1MAsset(Asset): ++ """ ++ Class for parsing top-1m.txt as an asset. ++ """ ++ def __init__(self, file=None): ++ self = Asset.__init__(self, file) ++ ++ def parse_line(self, line): ++ self = Asset.parse_line(self, line) ++ return line.split(',')[1].replace('\n','') ++ +class DNSTAsset(Asset): ++ """ ++ Creates DNS testing specific Assets. ++ """ + def __init__(self, file=None): - self = asset.__init__(self, file) ++ self = Asset.__init__(self, file) + +class DNST(Test): + def lookup(self, hostname, ns): ++ """ ++ Resolves a hostname through a DNS nameserver, ns, to the corresponding ++ IP address(es). ++ """ + res = resolver.Resolver(configure=False) + res.nameservers = [ns] + answer = res.query(hostname) + + ret = [] + + for data in answer: + ret.append(data.address) + + return ret + ++ def reverse_lookup(self, ip, ns): ++ """ ++ Attempt to do a reverse DNS lookup to determine if the control and exp ++ sets from a positive result resolve to the same domain, in order to ++ remove false positives due to GeoIP load balancing. ++ """ ++ res = resolver.Resolver(configure=False) ++ res.nameservers = [ns] ++ n = reversename.from_address(ip) ++ revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] ++ ++ return revn ++ + def experiment(self, *a, **kw): ++ """ ++ Compares the lookup() sets of the control and experiment groups. ++ """ + # this is just a dirty hack + address = kw['data'][0] + ns = kw['data'][1] + + config = self.config ++ ctrl_ns = config.tests.dns_control_server + + print "ADDRESS: %s" % address + print "NAMESERVER: %s" % ns + + exp = self.lookup(address, ns) - control = self.lookup(address, config.tests.dns_control_server) ++ control = self.lookup(address, ctrl_ns) ++ ++ result = [] + + if len(set(exp) & set(control)) > 0: - print "%s : no tampering on %s" % (address, ns) - return (address, ns, False) ++ print "Address %s has not tampered with on DNS server %s\n" % (address, ns) ++ result = (address, ns, exp, control, False) ++ return result + else: - print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control) - return (address, ns, exp, control, True) ++ print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) ++ result = (address, ns, exp, control, True) ++ ++ if config.tests.dns_reverse_lookup: ++ ++ exprevn = [self.reverse_lookup(ip, ns) for ip in exp] ++ ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) ++ for ip in control] ++ ++ if len(set(exprevn) & set(ctrlrevn)) > 0: ++ print "Further testing has eliminated this as a false positive." ++ else: ++ print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) ++ return result ++ ++ else: ++ print "\n" ++ return result + +def run(ooni): - """Run the test ++ """ ++ Run the test. + """ + config = ooni.config + urls = [] + - dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \ - config.tests.dns_experiment)) - dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \ ++ if (config.tests.dns_experiment == "top-1m.txt"): ++ dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, ++ config.tests.dns_experiment)) ++ else: ++ dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, ++ config.tests.dns_experiment)) ++ dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, + config.tests.dns_experiment_dns)) + + assets = [dns_experiment, dns_experiment_dns] + + dnstest = DNST(ooni) - ooni.logger.info("starting test") - dnstest.run(assets) - ooni.logger.info("finished") - ++ ooni.logger.info("Beginning dnstamper test...") ++ dnstest.run(assets, {'index': 1}) ++ ooni.logger.info("Dnstamper test completed!") + diff --cc ooni/oonitests/httphost.py index 6446e1f,0000000..25adff2 mode 100644,000000..100644 --- a/ooni/oonitests/httphost.py +++ b/ooni/oonitests/httphost.py @@@ -1,132 -1,0 +1,135 @@@ +""" + HTTP Host based filtering + ************************* + + This test detect HTTP Host field + based filtering. + It is used to detect censorship on + performed with Web Guard (used by + T-Mobile US). +""" +import os +from datetime import datetime +from gevent import monkey + +import urllib2 +import httplib +# WARNING! Using gevent's socket +# introduces the 0x20 DNS "feature". +# This will result is weird DNS requests +# appearing on the wire. +monkey.patch_socket() + - from BeautifulSoup import BeautifulSoup ++try: ++ from BeautifulSoup import BeautifulSoup ++except: ++ print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/" + +from plugoo.assets import Asset +from plugoo.tests import Test + +__plugoo__ = "HTTP Host" +__desc__ = "This detects HTTP Host field based filtering" + +class HTTPHostAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + return line.split(',')[1].replace('\n','') + +class HTTPHost(Test): + """ + The main Test class + """ + + def check_response(self, response): + soup = BeautifulSoup(response) + if soup.head.title.string == "Content Filtered": + # Response indicates censorship + return True + else: + # Response does not indicate censorship + return False + + + def is_censored(self, response): + if response: + soup = BeautifulSoup(response) + censored = self.check_response(response) + else: + censored = "unreachable" + return censored + + def urllib2_test(self, control_server, host): + req = urllib2.Request(control_server) + req.add_header('Host', host) + try: + r = urllib2.urlopen(req) + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + def httplib_test(self, control_server, host): + try: + conn = httplib.HTTPConnection(control_server) + conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) + conn.putheader("Host", host) + conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") + conn.endheaders() + r = conn.getresponse() + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + + def experiment(self, *a, **kw): + """ + Try to connect to the control server with + the specified host field. + """ + host = kw['data'] + control_server = kw['control_server'] + self.logger.info("Testing %s (%s)" % (host, control_server)) + + #censored = self.urllib2_test(control_server, host) + censored = self.httplib_test(control_server, host) + + self.logger.info("%s: %s" % (host, censored)) + return {'Time': datetime.now(), + 'Host': host, + 'Censored': censored} + + +def run(ooni): + """ + This is the function that will be called by OONI + and it is responsible for instantiating and passing + the arguments to the Test class. + """ + config = ooni.config + + # This the assets array to be passed to the run function of + # the test + assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \ + "top-1m.csv"))] + + # Instantiate the Test + thetest = HTTPHost(ooni) + ooni.logger.info("starting HTTP Host Test...") + # Run the test with argument assets + thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'}) + ooni.logger.info("finished.") + +

1 0

[ooni-probe/master] Add script to allow fast scaffolding of tests
by art＠torproject.org 31 May '12

31 May '12

commit 66b806472e57f8be90b0b94b5b4354677ec7102f Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 03:32:24 2012 +0200 Add script to allow fast scaffolding of tests --- ooni/plugins/scaffolding.py | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 66 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/scaffolding.py b/ooni/plugins/scaffolding.py new file mode 100755 index 0000000..fb7002d --- /dev/null +++ b/ooni/plugins/scaffolding.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +""" +This script should be used for creating the scaffolding for a test. +""" +import os +import sys + +test_template = """\"\"\" +This is a self genrated test created by scaffolding.py. +you will need to fill it up with all your necessities. +Safe hacking :). +\"\"\" +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from plugoo.tests import ITest, TwistedTest + +class %(testName)sArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index']] + +class %(testName)sTest(TwistedTest): + implements(IPlugin, ITest) + + shortName = "%(testShortname)s" + description = "%(testName)s" + requirements = None + options = %(testName)sArgs + blocking = True + + def control(self, experiment_result, args): + # What you return here ends up inside of the report. + return {} + + def experiment(self, args): + # What you return here gets handed as input to control + return {} + + def load_assets(self): + if self.local_options: + return {'asset': Asset(self.local_options['asset'])} + else: + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +%(testShortname)s = %(testName)sTest(None, None, None) +""" + +test_vars = {'testName': None, 'testShortname': None} +test_vars['testName'] = raw_input('Test Name: ') +test_vars['testShortname'] = raw_input("Test Short Name: ") + +fname = test_vars['testShortname']+'.py' + +if os.path.exists(fname): + print 'WARNING! File named "%s" already exists.' % fname + if raw_input("Do you wish to continue (y/N)? ").lower() != 'y': + print "gotcha! Dying.." + sys.exit(0) + +fp = open(fname, 'w') +fp.write(test_template % test_vars) +fp.close() +

1 0

[ooni-probe/master] Implement logging and reporting
by art＠torproject.org 31 May '12

31 May '12

commit a319db38669313aa8bd5a31f1c991a300434dfbc Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 04:42:55 2012 +0200 Implement logging and reporting * Port various tests to new model --- .gitignore | 2 +- ooni/__init__.py | 1 + ooni/log.py | 35 +++++++++ ooni/oonicli.py | 8 ++- ooni/plugins/blocking.py | 2 +- ooni/plugins/dropin.cache | 164 ------------------------------------------- ooni/plugins/httphost.py | 3 +- ooni/plugins/scaffolding.py | 66 ----------------- ooni/plugins/skel.py | 1 + ooni/plugoo/reports.py | 39 ++++------ ooni/plugoo/tests.py | 13 ++-- ooni/plugoo/work.py | 7 ++- ooni/scaffolding.py | 67 ++++++++++++++++++ 13 files changed, 143 insertions(+), 265 deletions(-) diff --git a/.gitignore b/.gitignore index 8ed47e9..553482d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ proxy-lists/ips.txt proxy-lists/italy-dns-ips.txt proxy-lists/italy-http-ips.txt private/* -ooni/plugins/dropin.cache +/ooni/plugins/dropin.cache diff --git a/ooni/__init__.py b/ooni/__init__.py new file mode 100644 index 0000000..dbb3824 --- /dev/null +++ b/ooni/__init__.py @@ -0,0 +1 @@ +__all__ = ['plugoo', 'utils', 'config', 'logo', 'lib'] diff --git a/ooni/log.py b/ooni/log.py new file mode 100644 index 0000000..bf6ea69 --- /dev/null +++ b/ooni/log.py @@ -0,0 +1,35 @@ +""" +OONI logging facility. +""" +import sys +import logging +import warnings + +from twisted.python import log + +# Logging levels +DEBUG = logging.DEBUG +INFO = logging.INFO +WARNING = logging.WARNING +ERROR = logging.ERROR +CRITICAL = logging.CRITICAL + +def _get_log_level(level): + if not level: + return INFO + else: + return level + +def start(logfile=None, loglevel=None, logstdout=True): + if log.defaultObserver: + loglevel = _get_log_level(loglevel) + logfile = logfile + file = open(logfile, 'a') if logfile else sys.stderr + log.startLogging(file, setStdout=logstdout) + msg("Started OONI") + +def msg(message, level=INFO, **kw): + log.msg(message, **kw) + +def err(message, **kw): + log.err(message, **kw) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index a0272fb..d612ba7 100755 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -19,9 +19,9 @@ from zope.interface.exceptions import BrokenMethodImplementation from zope.interface.verify import verifyObject from pprint import pprint -from ooni.plugoo import tests, work, assets +from ooni.plugoo import tests, work, assets, reports from ooni.logo import getlogo -from ooni import plugins +from ooni import plugins, log __version__ = "0.0.1-prealpha" @@ -52,13 +52,15 @@ def runTest(test, options, global_options): parallelism = int(global_options['parallelism']) worker = work.Worker(parallelism) test = plugoo[test].__class__ + report = reports.Report(global_options['output']) #if options['asset']: # print options['asset'] # asset = assets.Asset(options['asset']) # print asset + log.start(global_options['log'], 1) - wgen = work.WorkGenerator(test(options, global_options), + wgen = work.WorkGenerator(test(options, global_options, report), dict(options), start=options['resume']) diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py index bbcb1a2..4638834 100644 --- a/ooni/plugins/blocking.py +++ b/ooni/plugins/blocking.py @@ -8,7 +8,7 @@ from ooni.plugoo.tests import ITest, TwistedTest class BlockingArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] + ['shit', 'o', None, 'Other arguments']] class BlockingTest(TwistedTest): implements(IPlugin, ITest) diff --git a/ooni/plugins/dropin.cache b/ooni/plugins/dropin.cache deleted file mode 100755 index bd26c09..0000000 --- a/ooni/plugins/dropin.cache +++ /dev/null @@ -1,164 +0,0 @@ -(dp1 -S'skel' -p2 -ccopy_reg -_reconstructor -p3 -(ctwisted.plugin -CachedDropin -p4 -c__builtin__ -object -p5 -NtRp6 -(dp7 -S'moduleName' -p8 -S'plugins.skel' -p9 -sS'description' -p10 -NsS'plugins' -p11 -(lp12 -g3 -(ctwisted.plugin -CachedPlugin -p13 -g5 -NtRp14 -(dp15 -S'provided' -p16 -(lp17 -ctwisted.plugin -IPlugin -p18 -acplugoo.tests -ITest -p19 -asS'dropin' -p20 -g6 -sS'name' -p21 -S'skel' -p22 -sg10 -NsbasbsS'bridget' -p23 -g3 -(g4 -g5 -NtRp24 -(dp25 -g8 -S'ooni.plugins.bridget' -p26 -sg10 -Nsg11 -(lp27 -g3 -(g13 -g5 -NtRp28 -(dp29 -g16 -(lp30 -g18 -ag19 -asg20 -g24 -sg21 -S'bridget' -p31 -sg10 -NsbasbsS'blocking' -p32 -g3 -(g4 -g5 -NtRp33 -(dp34 -g8 -S'ooni.plugins.blocking' -p35 -sg10 -Nsg11 -(lp36 -g3 -(g13 -g5 -NtRp37 -(dp38 -g16 -(lp39 -g18 -acooni.plugoo.tests -ITest -p40 -asg20 -g33 -sg21 -S'blocking' -p41 -sg10 -NsbasbsS'httphost' -p42 -g3 -(g4 -g5 -NtRp43 -(dp44 -g8 -S'ooni.plugins.httphost' -p45 -sg10 -S'\n HTTP Host based filtering\n *************************\n\n This test detect HTTP Host field\n based filtering.\n' -p46 -sg11 -(lp47 -g3 -(g13 -g5 -NtRp48 -(dp49 -g16 -(lp50 -g18 -ag40 -asg20 -g43 -sg21 -S'httphost' -p51 -sg10 -NsbasbsS'tcpscan' -p52 -g3 -(g4 -g5 -NtRp53 -(dp54 -g8 -S'plugins.tcpscan' -p55 -sg10 -Nsg11 -(lp56 -g3 -(g13 -g5 -NtRp57 -(dp58 -g16 -(lp59 -g18 -ag40 -asg20 -g53 -sg21 -S'tcpscan' -p60 -sg10 -Nsbasbs. \ No newline at end of file diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index 17e2e11..9b4de22 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -89,6 +89,8 @@ class HTTPHostTest(TwistedTest): return censored def httplib_test(self, control_server, host): + censored = None + response = None try: conn = httplib.HTTPConnection(control_server) conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) @@ -100,7 +102,6 @@ class HTTPHostTest(TwistedTest): censored = self.is_censored(response) except Exception, e: censored = "Error! %s" % e - respopnse = None return (censored, response) diff --git a/ooni/plugins/scaffolding.py b/ooni/plugins/scaffolding.py deleted file mode 100755 index fb7002d..0000000 --- a/ooni/plugins/scaffolding.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -""" -This script should be used for creating the scaffolding for a test. -""" -import os -import sys - -test_template = """\"\"\" -This is a self genrated test created by scaffolding.py. -you will need to fill it up with all your necessities. -Safe hacking :). -\"\"\" -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from plugoo.tests import ITest, TwistedTest - -class %(testName)sArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index']] - -class %(testName)sTest(TwistedTest): - implements(IPlugin, ITest) - - shortName = "%(testShortname)s" - description = "%(testName)s" - requirements = None - options = %(testName)sArgs - blocking = True - - def control(self, experiment_result, args): - # What you return here ends up inside of the report. - return {} - - def experiment(self, args): - # What you return here gets handed as input to control - return {} - - def load_assets(self): - if self.local_options: - return {'asset': Asset(self.local_options['asset'])} - else: - return {} - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -%(testShortname)s = %(testName)sTest(None, None, None) -""" - -test_vars = {'testName': None, 'testShortname': None} -test_vars['testName'] = raw_input('Test Name: ') -test_vars['testShortname'] = raw_input("Test Short Name: ") - -fname = test_vars['testShortname']+'.py' - -if os.path.exists(fname): - print 'WARNING! File named "%s" already exists.' % fname - if raw_input("Do you wish to continue (y/N)? ").lower() != 'y': - print "gotcha! Dying.." - sys.exit(0) - -fp = open(fname, 'w') -fp.write(test_template % test_vars) -fp.close() - diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py index 00e4311..c5eec47 100644 --- a/ooni/plugins/skel.py +++ b/ooni/plugins/skel.py @@ -2,6 +2,7 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin from plugoo.tests import ITest, TwistedTest +from ooni import log class SkelArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index c099456..44634b8 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -2,9 +2,8 @@ import os from datetime import datetime import yaml -import logging import itertools -import gevent +from ooni import log class Report: """This is the ooni-probe reporting mechanism. It allows @@ -20,29 +19,27 @@ class Report: inbound connection and accept a stream of data (think of it as a `nc -l -p <port> > filename.txt`) """ - def __init__(self, ooni, - scp="127.0.0.1:22", - file="test.report", - tcp="127.0.0.1:9000"): + def __init__(self, file="test.report", + scp=None, + tcp=None): self.file = file self.tcp = tcp self.scp = scp - self.config = ooni.config.report - self.logger = ooni.logger + #self.config = ooni.config.report - if self.config.timestamp: - tmp = self.file.split('.') - self.file = '.'.join(tmp[:-1]) + "-" + \ - datetime.now().isoformat('-') + '.' + \ - tmp[-1] - print self.file + #if self.config.timestamp: + # tmp = self.file.split('.') + # self.file = '.'.join(tmp[:-1]) + "-" + \ + # datetime.now().isoformat('-') + '.' + \ + # tmp[-1] + # print self.file try: import paramiko except: self.scp = None - self.logger.warn("Could not import paramiko. SCP will not be disabled") + log.err("Could not import paramiko. SCP will not be disabled") def __call__(self, data): """ @@ -64,13 +61,9 @@ class Report: if self.scp: reports.append("scp") - jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports] - gevent.joinall(jobs) - ret = [] - for job in jobs: - #print job.value - ret.append(job.value) - return ret + #XXX make this non blocking + for report in reports: + self.send_report(dump, report) def file_report(self, data, file=None, mode='a+'): """ @@ -169,7 +162,7 @@ class Report: specified type. """ #print "Reporting %s to %s" % (data, type) - self.logger.info("Reporting to %s" % type) + log.msg("Reporting to %s" % type) getattr(self, type+"_report").__call__(data) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 75d0765..fd19479 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -10,6 +10,7 @@ import gevent from twisted.internet import reactor, defer, threads from twisted.python import failure +from ooni import log from ooni.plugoo import assets, work from ooni.plugoo.reports import Report @@ -142,10 +143,11 @@ class ITest(Interface): class TwistedTest(object): blocking = False - def __init__(self, local_options, global_options, ooninet=None): + def __init__(self, local_options, global_options, report, ooninet=None): self.local_options = local_options self.global_options = global_options self.assets = self.load_assets() + self.report = report #self.ooninet = ooninet def load_assets(self): @@ -167,7 +169,8 @@ class TwistedTest(object): result['end_time'] = self.end_time result['run_time'] = self.end_time - self.start_time result['control'] = control - print "FINISHED", result + log.msg("FINISHED %s" % result) + self.report(result) return result def _do_experiment(self, args): @@ -181,15 +184,15 @@ class TwistedTest(object): return self.d def control(self, result, args): - print "Doing control..." + log.msg("Doing control") return result def experiment(self, args): - print "Doing experiment" + log.msg("Doing experiment") return {} def startTest(self, args): self.start_time = datetime.now() - print "Starting test %s" % self.__class__ + log.msg("Starting test %s" % self.__class__) return self._do_experiment(args) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 55bbd1d..b9fc6b8 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -38,13 +38,18 @@ class Worker(object): asset, test, idx = workunit self._running += 1 actuald = test.startTest(asset).addBoth(self._run) + if isinstance(r, failure.Failure): r.trap() - print "Callback fired!" print r['start_time'] print r['end_time'] print r['run_time'] + + if self._running == 0 and not self._queued: + print "I am done." + reactor.stop() + return r def push(self, workunit): diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py new file mode 100755 index 0000000..b1d3f59 --- /dev/null +++ b/ooni/scaffolding.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +""" +This script should be used for creating the scaffolding for a test. +""" +import os +import sys +from ooni import log + +test_template = """\"\"\" +This is a self genrated test created by scaffolding.py. +you will need to fill it up with all your necessities. +Safe hacking :). +\"\"\" +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from plugoo.tests import ITest, TwistedTest + +class %(testName)sArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index']] + +class %(testName)sTest(TwistedTest): + implements(IPlugin, ITest) + + shortName = "%(testShortname)s" + description = "%(testName)s" + requirements = None + options = %(testName)sArgs + blocking = True + + def control(self, experiment_result, args): + # What you return here ends up inside of the report. + return {} + + def experiment(self, args): + # What you return here gets handed as input to control + return {} + + def load_assets(self): + if self.local_options: + return {'asset': Asset(self.local_options['asset'])} + else: + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +%(testShortname)s = %(testName)sTest(None, None, None) +""" + +test_vars = {'testName': None, 'testShortname': None} +test_vars['testName'] = raw_input('Test Name: ') +test_vars['testShortname'] = raw_input("Test Short Name: ") + +fname = os.path.join('plugins', test_vars['testShortname']+'.py') + +if os.path.exists(fname): + print 'WARNING! File named "%s" already exists.' % fname + if raw_input("Do you wish to continue (y/N)? ").lower() != 'y': + print "gotcha! Dying.." + sys.exit(0) + +fp = open(fname, 'w') +fp.write(test_template % test_vars) +fp.close() +

1 0

[ooni-probe/master] Move tests to proper directory
by art＠torproject.org 31 May '12

31 May '12

commit 59f1b4e82ccc906be419b35bfdc4cce61d69c869 Author: Arturo Filastò <hellais(a)torproject.org> Date: Tue May 29 01:45:41 2012 +0200 Move tests to proper directory --- ooni/tests/worker_test.py | 27 +++++++++++++++++++++++++++ unittest/tests.py | 27 --------------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/ooni/tests/worker_test.py b/ooni/tests/worker_test.py new file mode 100644 index 0000000..10887b2 --- /dev/null +++ b/ooni/tests/worker_test.py @@ -0,0 +1,27 @@ +from twisted.internet import reactor +from plugoo import work, tests + +class StupidAsset(object): + def __init__(self): + self.idx = 0 + + def __iter__(self): + return self + + def next(self): + if self.idx > 30: + raise StopIteration + self.idx += 1 + return self.idx + +wgen = work.WorkGenerator(StupidAsset, tests.StupidTest(None, None, None, None), {'bla': 'aaa'}, start=0) +worker = work.Worker() +for x in wgen: + print "------" + print "Work unit" + print "------" + worker.push(x) + print "------" + +reactor.run() + diff --git a/unittest/tests.py b/unittest/tests.py deleted file mode 100644 index 10887b2..0000000 --- a/unittest/tests.py +++ /dev/null @@ -1,27 +0,0 @@ -from twisted.internet import reactor -from plugoo import work, tests - -class StupidAsset(object): - def __init__(self): - self.idx = 0 - - def __iter__(self): - return self - - def next(self): - if self.idx > 30: - raise StopIteration - self.idx += 1 - return self.idx - -wgen = work.WorkGenerator(StupidAsset, tests.StupidTest(None, None, None, None), {'bla': 'aaa'}, start=0) -worker = work.Worker() -for x in wgen: - print "------" - print "Work unit" - print "------" - worker.push(x) - print "------" - -reactor.run() -

1 0

[ooni-probe/master] Rename oonicli to ooniprobe
by art＠torproject.org 31 May '12

31 May '12

commit 37d160cd14422b5e21707d97e1fc128fad6a9b5e Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 04:51:33 2012 +0200 Rename oonicli to ooniprobe --- ooni/oonicli.py | 117 ----------------------- ooni/ooniprobe.py | 272 ++++++++++++++++++++--------------------------------- 2 files changed, 101 insertions(+), 288 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py deleted file mode 100755 index d612ba7..0000000 --- a/ooni/oonicli.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -# -# oonicli -# ******* -# -# :copyright: (c) 2012 by Arturo Filastò -# :license: see LICENSE for more details. -# - -import sys - -from twisted.python import usage -from twisted.plugin import getPlugins -from twisted.internet import reactor - -from zope.interface.exceptions import BrokenImplementation -from zope.interface.exceptions import BrokenMethodImplementation -from zope.interface.verify import verifyObject -from pprint import pprint - -from ooni.plugoo import tests, work, assets, reports -from ooni.logo import getlogo -from ooni import plugins, log - -__version__ = "0.0.1-prealpha" - -def retrieve_plugoo(): - """ - Get all the plugins that implement the ITest interface and get the data - associated to them into a dict. - """ - interface = tests.ITest - d = {} - error = False - for p in getPlugins(interface, plugins): - try: - verifyObject(interface, p) - d[p.shortName] = p - except BrokenImplementation, bi: - print "Plugin Broken" - print bi - error = True - if error != False: - print "Plugin Loaded!" - return d - -plugoo = retrieve_plugoo() - -def runTest(test, options, global_options): - - parallelism = int(global_options['parallelism']) - worker = work.Worker(parallelism) - test = plugoo[test].__class__ - report = reports.Report(global_options['output']) - - #if options['asset']: - # print options['asset'] - # asset = assets.Asset(options['asset']) - # print asset - log.start(global_options['log'], 1) - - wgen = work.WorkGenerator(test(options, global_options, report), - dict(options), - start=options['resume']) - - for x in wgen: - worker.push(x) - - reactor.run() - -class Options(usage.Options): - tests = plugoo.keys() - subCommands = [] - for test in tests: - subCommands.append([test, None, plugoo[test].options, "Run the %s test" % test]) - - optFlags = [ - #['remote', 'r', "If the test should be run remotely (not supported)"], - #['status', 'x', 'Show current state'], - #['restart', 'r', 'Restart OONI'] - ] - - optParameters = [ - ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], - #['target-node', 't', 'localhost:31415', 'Select target node'], - ['output', 'o', 'report.log', "Specify output report file"], - ['log', 'l', 'oonicli.log', "Specify output log file"], - #['password', 'p', 'opennetwork', "Specify the password for authentication"], - ] - - def opt_version(self): - """ - Display OONI version and exit. - """ - print "OONI version:", __version__ - sys.exit(0) - - def __str__(self): - """ - Hack to get the sweet ascii art into the help output and replace the - strings "Commands" with "Tests". - """ - return getlogo() + '\n' + self.getSynopsis() + '\n' + \ - self.getUsage(width=None).replace("Commands:", "Tests:") - -config = Options() -config.parseOptions() - -if not config.subCommand: - print "Error! No Test Specified." - config.opt_help() - sys.exit(1) - -runTest(config.subCommand, config.subOptions, config) - - diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 3e8fbe7..d75b9b9 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -1,191 +1,121 @@ #!/usr/bin/env python # -*- coding: UTF-8 -""" - ooniprobe - ********** +# +# ooniprobe +# ********* +# +# Open Observatory of Network Interference +# +# "The Net interprets censorship as damage and routes around it." +# - John Gilmore; TIME magazine (6 December 1993) +# +# The goal of ooni-probe is to collect data about censorship around +# the world. +# +# :copyright: (c) 2012 by Arturo Filastò +# :license: see LICENSE for more details. +# - Open Observatory of Network Interference - - "The Net interprets censorship as damage and routes around it." - - John Gilmore; TIME magazine (6 December 1993) - - The goal of ooni-probe is to collect data about censorship around - the world. - - :copyright: (c) 2012 by Arturo Filastò - :license: see LICENSE for more details. -""" - -import imp -import os import sys -import argparse +from twisted.python import usage +from twisted.plugin import getPlugins +from twisted.internet import reactor + +from zope.interface.exceptions import BrokenImplementation +from zope.interface.exceptions import BrokenMethodImplementation +from zope.interface.verify import verifyObject from pprint import pprint -import plugoo -from utils import Storage, parse_asset, import_test, get_logger -from config import Config -from logo import getlogo +from ooni.plugoo import tests, work, assets, reports +from ooni.logo import getlogo +from ooni import plugins, log + +__version__ = "0.0.1-prealpha" -class ooni(object): +def retrieve_plugoo(): """ - ooni-probe is a suite designed to run tests on your - network to detect censorship. - This is the main class that is used to start ooni probe - select the assets and run tests. + Get all the plugins that implement the ITest interface and get the data + associated to them into a dict. """ - def __init__(self): - self.config = Storage() - self.config.main = Config("main") - self.config.tests = Config("tests") - self.config.report = Config("report") - - self.logger = get_logger(self.config.main) - - self.logger.info("Started ooni-probe") - - self.assets = [] - self.get_assets() - - self.tests = Storage() - #self.load_tests() - - self.runtests = self.config.tests.run.split(",") - - - def get_assets(self): + interface = tests.ITest + d = {} + error = False + for p in getPlugins(interface, plugins): + try: + verifyObject(interface, p) + d[p.shortName] = p + except BrokenImplementation, bi: + print "Plugin Broken" + print bi + error = True + if error != False: + print "Plugin Loaded!" + return d + +plugoo = retrieve_plugoo() + +def runTest(test, options, global_options): + + parallelism = int(global_options['parallelism']) + worker = work.Worker(parallelism) + test = plugoo[test].__class__ + report = reports.Report(global_options['output']) + + log.start(global_options['log'], 1) + + wgen = work.WorkGenerator(test(options, global_options, report), + dict(options), + start=options['resume']) + + for x in wgen: + worker.push(x) + + reactor.run() + +class Options(usage.Options): + tests = plugoo.keys() + subCommands = [] + for test in tests: + subCommands.append([test, None, plugoo[test].options, "Run the %s test" % test]) + + optFlags = [ + #['remote', 'r', "If the test should be run remotely (not supported)"], + #['status', 'x', 'Show current state'], + #['restart', 'r', 'Restart OONI'] + ] + + optParameters = [ + ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], + #['target-node', 't', 'localhost:31415', 'Select target node'], + ['output', 'o', 'report.log', "Specify output report file"], + ['log', 'l', 'oonicli.log', "Specify output log file"], + #['password', 'p', 'opennetwork', "Specify the password for authentication"], + ] + + def opt_version(self): """ - Parse all the assets in the asset directory. - Assets can optionaly contain the ooni-probe asset file - format: #:<something> <something_else>, that will then - be used to render the asset details to the user. - It is also possible to have an asset file link to multiple - other files. + Display OONI version and exit. """ - for root, dir, files in os.walk(self.config.main.assetdir): - for name in files: - asset = os.path.join(root, name) - self.assets.append(parse_asset(asset)) + print "OONI version:", __version__ + sys.exit(0) - def list_assets(self): - """ - Enumerate all the assets in the directory specified - in the config file - """ - print "[-] There are a total of %s assets loaded" % len(self.assets) - for asset in self.assets: - print " name: %s" % asset.name - if asset.desc: - print " description: %s" % asset.desc - if asset.files: - print " files: %s" % asset.files - if asset.tests: - print " tests: %s" % asset.tests - print "" - - def load_tests(self): + def __str__(self): """ - Iterate through the plugoos inside the folder specified by the - config file and instantiate them. + Hack to get the sweet ascii art into the help output and replace the + strings "Commands" with "Tests". """ - pluginfiles = [fname[:-3] for fname in os.listdir(self.config.main.testdir)\ - if fname.endswith(".py")] - for fname in pluginfiles: - test = Storage() - test_name = fname - if not self.config.main.testdir in sys.path: - sys.path.insert(0, self.config.main.testdir) - #print "Fname: %s\n__import__(%s)" % (fname, fname) - #print sys.path - module = __import__(fname) - try: - test.name = module.__plugoo__ - test.desc = module.__desc__ - test.module = module - except Exception, e: - self.logger.warning("Soft fail %s", e) - test.name = test_name - test.desc = "" - test.module = module - - try: - self.tests[test_name] = test - except Exception, e: - print "Failed to load the test %s %s" % (name, e) - - def list_tests(self): - """ - Print the loaded plugoonis to screen - """ - self.load_tests() - print "[-] There are a total of %s tests available" % len(self.tests) - for name, test in self.tests.items(): - print " name: %s" % name - if test.name: - print " long name: %s" % test.name - if test.desc: - print " description: %s" % test.desc - print "" - - - def run_tests(self): - """ - Run all the tests that have been loaded - """ - self.load_tests() - for name in self.runtests: - self.logger.info("running %s" % name) - try: - self.tests[name].module.run(self) - except Exception, e: - self.logger.error("ERR: %s" % e) - - def run_test(self, test, asset=None): - """ - Run a single test - """ - self.load_tests() - if asset: - self.tests[test].module.run(self, asset) - else: - self.tests[test].module.run(self) - -if __name__ == "__main__": - o = ooni() - - parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, - description=getlogo() + - '\n\n Open Observatory of Network Interference.') - - parser.add_argument('-t', '--list-tests', help='List all the available tests', - action='store_true', dest='list_tests') - - parser.add_argument('-l', '--list-assets', help='List all the assets', - action='store_true', dest='list_assets') - - parser.add_argument('-r', '--run', help='Run a certain test', action='store') - - parser.add_argument('-a', '--asset', help='Use this asset for the test', - action='store') - - parser.add_argument('--runall', help='Run all the tests in the config', - action='store_true') - - args = parser.parse_args() - - if args.list_tests: - o.list_tests() + return getlogo() + '\n' + self.getSynopsis() + '\n' + \ + self.getUsage(width=None).replace("Commands:", "Tests:") - if args.list_assets: - o.list_assets() +config = Options() +config.parseOptions() - if args.run: - o.run_test(args.run, args.asset) +if not config.subCommand: + print "Error! No Test Specified." + config.opt_help() + sys.exit(1) - elif args.runall: - o.run_tests() +runTest(config.subCommand, config.subOptions, config) - elif len(sys.argv) < 2: - parser.print_help()

1 0