commit 0080bf6555e4b55d0391c1c2f3ad41c83428f7e9 Author: Arturo Filastò arturo@filasto.net Date: Thu Aug 23 17:54:18 2012 +0200

Rename new bridget to new bridget --- ooni/assets/bridgetests.txt | 6 +- ooni/ooniprobe.py | 14 +++--- ooni/plugins/bridget.py | 76 ------------------------------- ooni/plugins/domclass.py | 37 ++++++++++++--- ooni/plugins/new_bridget.py | 105 +++++++++++++++++++++++++++++++++++++++++++ ooni/plugoo/interface.py | 1 + ooni/plugoo/reports.py | 49 ++++++++++++++++++++ ooni/plugoo/tests.py | 1 + 8 files changed, 196 insertions(+), 93 deletions(-)

diff --git a/ooni/assets/bridgetests.txt b/ooni/assets/bridgetests.txt index c837988..5519eea 100644 --- a/ooni/assets/bridgetests.txt +++ b/ooni/assets/bridgetests.txt @@ -1,3 +1,3 @@ -213.151.89.102:9001 -108.166.106.156:443 -217.150.224.213:443 +88.130.86.191:443 +195.74.237.236:9001 +127.0.0.1:9050 diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 4b5c086..f80669e 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -76,17 +76,18 @@ def runTest(test, options, global_options, reactor=reactor): resume = 0 if not options: options = {} + if 'resume' in options: resume = options['resume']

test = test_class(options, global_options, report, reactor=reactor) + if test.tool: test.runTool() - return

if test.ended: - print "Ending prematurely" - return + print "Ending test" + return None

wgen = work.WorkGenerator(test, dict(options), @@ -94,6 +95,8 @@ def runTest(test, options, global_options, reactor=reactor): for x in wgen: worker.push(x)

+ return True + class Options(usage.Options): tests = plugoo.keys() subCommands = [] @@ -138,7 +141,6 @@ if __name__ == "__main__": config.opt_help() sys.exit(1)

- runTest(config.subCommand, config.subOptions, config) - - reactor.run() + if runTest(config.subCommand, config.subOptions, config): + reactor.run()

diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py deleted file mode 100644 index 22ff9a0..0000000 --- a/ooni/plugins/bridget.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -This is a self genrated test created by scaffolding.py. -you will need to fill it up with all your necessities. -Safe hacking :). -""" -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import reactor - -from ooni.utils import log -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset - -class bridgetArgs(usage.Options): - optParameters = [['bridges', 'b', None, 'List of bridges to scan'], - ['relays', 'f', None, 'List of relays to scan'], - ['resume', 'r', 0, 'Resume at this index']] - -class bridgetTest(OONITest): - implements(IPlugin, ITest) - - shortName = "bridget" - description = "bridget" - requirements = None - options = bridgetArgs - blocking = False - - def experiment(self, args): - log.msg("Doing test") - # What you return here gets handed as input to control - from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState - from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor - def updates(prog, tag, summary): - log.msg("%d%%: %s" % (prog, summary)) - return - - def setup_failed(args): - log.msg("Setup Failed.") - report.update({'failed': args}) - return report - - def setup_complete(proto): - log.msg("Setup Complete.") - report.update({'success': args}) - return report - - config = TorConfig() - import random - config.SocksPort = random.randint(1024, 2**16) - config.ControlPort = random.randint(1024, 2**16) - - if 'bridge' in args: - config.UseBridges = 1 - config.Bridge = args['bridge'] - config.save() - print config.create_torrc() - report = {'tor_config': config.config} - log.msg("Starting Tor") - d = launch_tor(config, self.reactor, progress_updates=updates) - d.addCallback(setup_complete) - d.addErrback(setup_failed) - return d - - def load_assets(self): - assets = {} - if self.local_options: - if self.local_options['bridges']: - assets.update({'bridge': Asset(self.local_options['bridges'])}) - elif self.local_options['relays']: - assets.update({'relay': Asset(self.local_options['relay'])}) - return assets - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -bridget = bridgetTest(None, None, None) diff --git a/ooni/plugins/domclass.py b/ooni/plugins/domclass.py index cdcd508..446b9e4 100644 --- a/ooni/plugins/domclass.py +++ b/ooni/plugins/domclass.py @@ -1,4 +1,4 @@ -# +# -*- coding: utf-8 # # domclass # ******** @@ -28,6 +28,12 @@ # probability matrix B. #

+try: + import numpy +except: + print "Error numpy not installed!" + +import yaml from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin @@ -40,7 +46,7 @@ class domclassArgs(usage.Options): optParameters = [['output', 'o', None, 'Output to write'], ['file', 'f', None, 'Corpus file'], ['fileb', 'b', None, 'Corpus file'], - ['asset', 'a', None, 'URL List'], + ['urls', 'u', None, 'URL List'], ['resume', 'r', 0, 'Resume at this index']]

# All HTML4 tags @@ -80,7 +86,6 @@ def compute_probability_matrix(dataset): :dataset: an array of pairs representing the parent child relationships. """ import itertools - import numpy ret = {} matrix = numpy.zeros((len(thetags) + 1, len(thetags) + 1))

@@ -155,15 +160,16 @@ class domclassTest(HTTPTest): #tool = True

def runTool(self): - import yaml, numpy site_a = readDOM(filename=self.local_options['file']) site_b = readDOM(filename=self.local_options['fileb']) + a = {} a['matrix'] = compute_probability_matrix(site_a) - a['eigen'] = compute_eigenvalue(a['matrix']) + a['eigen'] = compute_eigenvalues(a['matrix'])

self.result['eigenvalues'] = a['eigen'] + b = {} b['matrix'] = compute_probability_matrix(site_b) - b['eigen'] = compute_eigenvalue(b['matrix']) + b['eigen'] = compute_eigenvalues(b['matrix'])

#print "A: %s" % a #print "B: %s" % b @@ -171,13 +177,15 @@ class domclassTest(HTTPTest): correlation /= numpy.linalg.norm(a['eigen'])*numpy.linalg.norm(b['eigen']) correlation = (correlation + 1)/2 print "Corelation: %s" % correlation + self.end() + return a

def processResponseBody(self, data): - import yaml, numpy site_a = readDOM(data) #site_b = readDOM(self.local_options['fileb']) + a = {} a['matrix'] = compute_probability_matrix(site_a) - a['eigen'] = compute_eigenvalue(a['matrix']) + a['eigen'] = compute_eigenvalues(a['matrix'])

if len(data) == 0: @@ -191,4 +199,17 @@ class domclassTest(HTTPTest): print "A: %s" % a return a['eigen']

+ def load_assets(self): + if self.local_options: + if self.local_options['file']: + self.tool = True + return {} + elif self.local_options['urls']: + return {'url': Asset(self.local_options['urls'])} + else: + self.end() + return {} + else: + return {} + domclass = domclassTest(None, None, None) diff --git a/ooni/plugins/new_bridget.py b/ooni/plugins/new_bridget.py new file mode 100644 index 0000000..3e4db56 --- /dev/null +++ b/ooni/plugins/new_bridget.py @@ -0,0 +1,105 @@ +""" +This is a self genrated test created by scaffolding.py. +you will need to fill it up with all your necessities. +Safe hacking :). +""" +from exceptions import Exception +from datetime import datetime +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import reactor, task + +from ooni.utils import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + +from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState +from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor + +class bridgetArgs(usage.Options): + optParameters = [['bridges', 'b', None, 'List of bridges to scan'], + ['relays', 'f', None, 'List of relays to scan'], + ['resume', 'r', 0, 'Resume at this index'], + ['timeout', 't', 5, 'Timeout in seconds after which to consider a bridge not working'] + ] + +class bridgetTest(OONITest): + implements(IPlugin, ITest) + + shortName = "bridget" + description = "bridget" + requirements = None + options = bridgetArgs + blocking = False + + def experiment(self, args): + log.msg("Doing test") + last_update = datetime.now() + tor_log = [] + + def check_timeout(): + log.msg("Checking for timeout") + time_since_update = datetime.now() - last_update + if time_since_update.seconds > self.local_options['timeout']: + log.msg("Timed out when connecting to %s" % args) + l.stop() + self.result['reason'] = 'timeout' + d.errback(args) + return + + def updates(prog, tag, summary): + tor_log.append((prog, tag, summary)) + last_update = datetime.now() + log.msg("%d%%: %s" % (prog, summary)) + + def setup_failed(failure): + log.msg("Setup Failed.") + if not self.result['reason']: + self.result['reason'] = 'unknown' + self.result['input'] = args + self.result['result'] = 'failed' + self.result['tor_log'] = tor_log + return + + def setup_complete(proto): + log.msg("Setup Complete.") + self.result['input'] = args + self.result['result'] = 'success' + return + + config = TorConfig() + import random + config.SocksPort = random.randint(1024, 2**16) + config.ControlPort = random.randint(1024, 2**16) + + if 'bridge' in args: + config.UseBridges = 1 + config.Bridge = args['bridge'] + + config.save() + + print config.config + self.result['tor_config'] = config.config + log.msg("Starting Tor connecting to %s" % args['bridge']) + + l = task.LoopingCall(check_timeout) + l.start(1.0) + + d = launch_tor(config, self.reactor, control_port=config.ControlPort, progress_updates=updates) + d.addCallback(setup_complete) + d.addErrback(setup_failed) + return d + + def load_assets(self): + assets = {} + if self.local_options: + if self.local_options['bridges']: + assets.update({'bridge': Asset(self.local_options['bridges'])}) + elif self.local_options['relays']: + assets.update({'relay': Asset(self.local_options['relay'])}) + return assets + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +bridget = bridgetTest(None, None, None) diff --git a/ooni/plugoo/interface.py b/ooni/plugoo/interface.py index d00e70b..6dc83a0 100644 --- a/ooni/plugoo/interface.py +++ b/ooni/plugoo/interface.py @@ -53,3 +53,4 @@ class ITest(Interface): be created. A report will be written. """

+ diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index a543151..1bfdac0 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -93,4 +93,53 @@ class Report: log.msg("Reporting to %s" % type) getattr(self, type+"_report").__call__(data)

+class NewReport(object): + filename = 'report.log' + startTime = None + endTime = None + testName = None + ipAddr = None + asnAddr = None + + def _open(): + self.fp = open(self.filename, 'a+') + + @property + def header(): + pretty_date = date.pretty_date() + report_header = "# OONI Probe Report for Test %s\n" % self.testName + report_header += "# %s\n\n" % pretty_date + test_details = {'start_time': self.startTime, + 'asn': asnAddr, + 'test_name': self.testName, + 'addr': ipAddr} + report_header += yaml.dump([test_details]) + return report_header + + def create(): + """ + Create a new report by writing it's header. + """ + self.fp = open(self.filename, 'w+') + self.fp.write(self.header) + + def exists(): + """ + Returns False if the file does not exists. + """ + return os.path.exists(self.filename) + + def write(data): + """ + Write a report to the file. + + :data: python data structure to be written to report. + """ + if not self.exists(): + self.create() + else: + self._open() + yaml_encoded_data = yaml.dump([data]) + self.fp.write(yaml_encoded_data) + self.fp.close()

diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 7c190eb..5fad85e 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -99,6 +99,7 @@ class OONITest(object):

self.d.addCallback(self.control, args) self.d.addCallback(self.finished) + self.d.addErrback(self.finished) return self.d

def control(self, result, args):