- tor-commits - lists.torproject.org

[ooni-probe/master] Edit readme
by art＠torproject.org 30 May '12

30 May '12

commit 5b682fc5b140136560c36b78b573b4a5b3245f84 Author: Arturo Filastò <hellais(a)torproject.org> Date: Mon Apr 30 18:36:16 2012 +0200 Edit readme --- README | 17 +++++++++++++++-- plugoo/tests.py | 16 +++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/README b/README index 476aa40..75262bf 100644 --- a/README +++ b/README @@ -3,8 +3,21 @@ ooni-probe - Open Observatory of Network Interference "The Net interprets censorship as damage and routes around it." - John Gilmore; TIME magazine (6 December 1993) -The goal of ooni-probe is to collect data about censorship around -the world. +OONI, the Open Observatory of Network Interference, is a global observation +network which aims is to collect high quality data using open methodologies, +using Free and Open Source Software (FL/OSS) to share observations and data +about the various types, methods, and amounts of network tampering in the world. + +With the belief that unfettered access to information is a intrinsic human right, +OONI seeks to observe levels of surveillance, censorship, and network discrimination +in order for people worldwide to have a clearer understanding of the ways in +which their access to information is filtered. + +The end goal of OONI is to collect data which can show an accurate +topology of network interference and censorship. Through this topology, it will be +possible to see what the internet looks like from nearly any location, including +what sites are censored, or have been tampered with, and by whom. We're calling +it filternet. OONI uses open methodologies and the data will be provided in raw format to allow any researcher to indipendently draw their conclusions diff --git a/plugoo/tests.py b/plugoo/tests.py index fc1d033..9a9f1de 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -111,13 +111,15 @@ class Test: else: self.logger.error("No Assets! Dying!") -class WorkUnit: +class WorkUnit(object): """ This is an object responsible for completing WorkUnits it will return its result in a deferred. The execution of a unit of work should be Atomic. + Reporting to the OONI-net happens on completion of a Unit of Work. + @Node node: This represents the node associated with the Work Unit @Asset asset: This is the asset associated with the Work Unit @Test test: This represents the Test to be with the specified assets @@ -151,6 +153,11 @@ class ITest(Interface): This interface represents an OONI test. It fires a deferred on completion. """ + shortName = Attribute("""A short user facing description for this test""") + description = Attribute("""A string containing a longer description for the test""") + + requirements = Attribtue("""What is required to run this this test, for example raw socket access or UDP or TCP""") + deferred = Attribute("""This will be fired on test completion""") node = Attribute("""This represents the node that will run the test""") arguments = Attribute("""These are the arguments to be passed to the test for it's execution""") @@ -160,9 +167,12 @@ class ITest(Interface): Launches the Test with the specified arguments on a node. """ -class WorkGenerator: +class WorkGenerator(object): """ Factory responsible for creating units of work. + + This shall be run on the machine running OONI-cli. The returned WorkUnits + can either be run locally or on a remote OONI Node or Network Node. """ node = LocalNode size = 10 @@ -230,7 +240,7 @@ class HTTPRequestTest(HTTPClient): self.censorship = False return response -class TwistedTestFactory: +class TwistedTestFactory(object): test = TwistedTest

1 0

[ooni-probe/master] Adjust reporting format
by art＠torproject.org 30 May '12

30 May '12

commit d3817c8d3236bf581e74c2fb7a04884e2f5c4c60 Author: Arturo Filastò <hellais(a)gmail.com> Date: Sun Apr 22 12:22:40 2012 +0200 Adjust reporting format --- plugoo/reports.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/plugoo/reports.py b/plugoo/reports.py index 90de498..c099456 100644 --- a/plugoo/reports.py +++ b/plugoo/reports.py @@ -50,8 +50,9 @@ class Report: data to the reporting system """ #print "Writing report(s)" - dump = '--- \n' - dump += yaml.dump(data) + #dump = '--- \n' + dump = yaml.dump([data]) + #dump += yaml.dump(data) reports = [] if self.file:

1 0

[ooni-probe/master] Start mocking out the classes for tests
by art＠torproject.org 30 May '12

30 May '12

commit 766eecf7b392446bad67ca657d8429eddcad0992 Author: Arturo Filastò <hellais(a)gmail.com> Date: Wed Apr 11 02:05:51 2012 -0700 Start mocking out the classes for tests --- plugoo/tests.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 48 insertions(+), 0 deletions(-) diff --git a/plugoo/tests.py b/plugoo/tests.py index 3d6b36b..08d32a8 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -111,4 +111,52 @@ class Test: else: self.logger.error("No Assets! Dying!") +class TwistedTest: + pass + + +class WorkUnit: + pass + +class WorkUnitFactory: + + asset_looped = False + rules = None + + def __init__(self, rules, assets, nodes): + self.rules = rules + self.nodes = nodes + self.nodes_gen = nodes() + + def next(self): + if asset_looped: + node = self.nodes() + yield WorkUnit(assets, node) + + +class TwistedTestFactory: + + test = TwistedTest + assets = None + + def __init__(self, assets, nodes, + rule="*", idx=0): + """ + """ + self.assets = assets + self.nodes = nodes + self.rule = rule + self.idx = idx + + def process_rules(self): + + + def build_test(self): + """ + Returns a TwistedTest instance + """ + workunit = self.workunit.next() + t = self.test(workunit) + t.factory = self + return t

1 0

[ooni-probe/master] Continue work on Worker and Test interfaces
by art＠torproject.org 30 May '12

30 May '12

commit 348a6d108936b998dbef1b626047bbf600d1eb99 Author: Arturo Filastò <hellais(a)gmail.com> Date: Fri Apr 20 16:25:40 2012 -0400 Continue work on Worker and Test interfaces --- plugoo/tests.py | 140 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 files changed, 116 insertions(+), 24 deletions(-) diff --git a/plugoo/tests.py b/plugoo/tests.py index 08d32a8..fc1d033 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -106,57 +106,149 @@ class Test: #print "JOB VAL: %s" % job.value self.logger.info("Writing report(s)") self.report(job.value) - job.kill() + job.kill() jobs = [] else: self.logger.error("No Assets! Dying!") -class TwistedTest: - pass +class WorkUnit: + """ + This is an object responsible for completing WorkUnits it will + return its result in a deferred. + The execution of a unit of work should be Atomic. -class WorkUnit: - pass + @Node node: This represents the node associated with the Work Unit + @Asset asset: This is the asset associated with the Work Unit + @Test test: This represents the Test to be with the specified assets + @ivar arguments: These are the extra attributes to be passsed to the Test + """ + + node = None + asset = None + test = None + arguments = None + + def __init__(self, node, asset, test, arguments): + self.assetGenerator = asset() + self.Test = test + self.node = node + self.arguments = arguments + + def next(): + """ + Launches the Unit of Work with the specified assets on the node. + """ + try: + asset = self.assetGenerator.next() + yield self.Test(asset, self.node, self.arguments) + except StopIteration: + raise StopIteration -class WorkUnitFactory: - asset_looped = False - rules = None +class ITest(Interface): + """ + This interface represents an OONI test. It fires a deferred on completion. + """ + + deferred = Attribute("""This will be fired on test completion""") + node = Attribute("""This represents the node that will run the test""") + arguments = Attribute("""These are the arguments to be passed to the test for it's execution""") + + def startTest(): + """ + Launches the Test with the specified arguments on a node. + """ + +class WorkGenerator: + """ + Factory responsible for creating units of work. + """ + node = LocalNode + size = 10 - def __init__(self, rules, assets, nodes): - self.rules = rules - self.nodes = nodes - self.nodes_gen = nodes() + def __init__(self, assets): + self.assets = assets() def next(self): - if asset_looped: - node = self.nodes() - yield WorkUnit(assets, node) + # Plank asset + p_asset = [] + for i in xrange(0, self.size): + p_asset.append(self.assets.next()) + yield WorkUnit(p_asset) + +def spawnDeferredTests(workunit, n): + def callback(result): + pass + + def errback(reason): + pass + + # XXX find more elegant solution to having 2 generators + workgenA = WorkGenerator(assets) + + for workunit in workgen: + deferredList = [] + workunitB = workunit + for i in range(n): + try: + test = workunit.next() + except StopIteration: + pass + + deferred = test.deferred + deferred.addCallback(callback).addErrback(errback) + + deferredList.append(deferred) + test.startTest() + + +class HTTPRequestTest(HTTPClient): + """ + This is an example of how I would like to be able to write a test. + + *BEWARE* this actually does not currently work, it's just an example of the + kind of API that I am attempting to achieve to simplify the writing of + tests. + """ + implements(ITest) + + def startTest(): + # The response object should also contain the request + """ + response = {'response': {'headers': ..., 'content': ..., + 'runtime': ..., 'timestamp': ...}, + 'request': {'headers': ..., 'content', 'timestamp', ...} + } + """ + response = self.http_request(address, headers) + if response.headers['content'].matches("Some string"): + self.censorship = True + return response + else: + self.censorship = False + return response class TwistedTestFactory: test = TwistedTest - assets = None - def __init__(self, assets, nodes, - rule="*", idx=0): + def __init__(self, assets, node, + idx=0): """ """ self.assets = assets - self.nodes = nodes - self.rule = rule + self.node = node self.idx = idx - - def process_rules(self): - + self.workunit = WorkUnitFactory(assets) def build_test(self): """ Returns a TwistedTest instance """ workunit = self.workunit.next() - t = self.test(workunit) + t = self.test(node, workunit) t.factory = self return t

1 0

[ooni-probe/master] Continue implementing twisted version of OONI
by art＠torproject.org 30 May '12

30 May '12

commit 113c237d387ac9c3984d99f908dc914d6f18d2a8 Author: Arturo Filastò <hellais(a)torproject.org> Date: Wed May 2 16:53:54 2012 +0200 Continue implementing twisted version of OONI --- oonicli.py | 57 +++++++++++++++++++ ooniprobe.py | 3 +- plugins/__init__.py | 3 + plugins/dropin.cache | 48 ++++++++++++++++ plugins/skel.py | 22 ++++++++ plugoo/interface.py | 2 + plugoo/nodes.py | 12 +++-- plugoo/tests.py | 135 +++++++++++++++------------------------------- plugoo/work.py | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++ plugoo/workers.py | 36 ------------ skel.py | 10 ++++ unittest/tests.py | 27 +++++++++ 12 files changed, 368 insertions(+), 133 deletions(-) diff --git a/oonicli.py b/oonicli.py new file mode 100755 index 0000000..d77ad0f --- /dev/null +++ b/oonicli.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 +# +# oonicli +# ******* +# +# :copyright: (c) 2012 by Arturo Filastò +# :license: see LICENSE for more details. +# + +from plugoo import tests + +from twisted.python import usage + +from twisted.plugin import getPlugins + +from zope.interface.exceptions import BrokenImplementation +from zope.interface.exceptions import BrokenMethodImplementation +from zope.interface.verify import verifyObject +import plugins + +def retrieve_plugoo(): + interface = tests.ITest + d = {} + error = False + for p in getPlugins(interface, plugins): + try: + verifyObject(interface, p) + d[p.shortName] = p + except BrokenImplementation, bi: + print "Plugin Broke" + print bi + error = True + except BrokenMethodImplementation, bmi: + print "Plugin Broke" + error = True + if error != False: + print "Plugin Loaded!" + return d + +plugoo = retrieve_plugoo() + +class Options(usage.Options): + tests = plugoo.keys() + subCommands = [] + for test in tests: + subCommands.append([test, None, plugoo[test].arguments, "Run the %s test" % test]) + + optParameters = [ + ['status', 's', 0, 'Show current state'], + ['restart', 'r', None, 'Restart OONI'], + ['node', 'n', 'localhost:31415', 'Select target node'] + ] + +config = Options() +config.parseOptions() + diff --git a/ooniprobe.py b/ooniprobe.py index 9d9c0ac..3e8fbe7 100755 --- a/ooniprobe.py +++ b/ooniprobe.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: UTF-8 """ - ooni-probe + ooniprobe ********** Open Observatory of Network Interference @@ -152,7 +152,6 @@ class ooni(object): self.tests[test].module.run(self) if __name__ == "__main__": - o = ooni() parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, diff --git a/plugins/__init__.py b/plugins/__init__.py new file mode 100644 index 0000000..ddb8691 --- /dev/null +++ b/plugins/__init__.py @@ -0,0 +1,3 @@ +from twisted.plugin import pluginPackagePaths +__path__.extend(pluginPackagePaths(__name__)) +__all__ = [] diff --git a/plugins/dropin.cache b/plugins/dropin.cache new file mode 100755 index 0000000..7bc3990 --- /dev/null +++ b/plugins/dropin.cache @@ -0,0 +1,48 @@ +(dp1 +S'skel' +p2 +ccopy_reg +_reconstructor +p3 +(ctwisted.plugin +CachedDropin +p4 +c__builtin__ +object +p5 +NtRp6 +(dp7 +S'moduleName' +p8 +S'plugins.skel' +p9 +sS'description' +p10 +NsS'plugins' +p11 +(lp12 +g3 +(ctwisted.plugin +CachedPlugin +p13 +g5 +NtRp14 +(dp15 +S'provided' +p16 +(lp17 +ctwisted.plugin +IPlugin +p18 +acplugoo.tests +ITest +p19 +asS'dropin' +p20 +g6 +sS'name' +p21 +S'skel' +p22 +sg10 +Nsbasbs. \ No newline at end of file diff --git a/plugins/skel.py b/plugins/skel.py new file mode 100644 index 0000000..d27932a --- /dev/null +++ b/plugins/skel.py @@ -0,0 +1,22 @@ +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from plugoo.tests import ITest + +class SkelArgs(usage.Options): + optParameters = [['assets', 'a', None, 'Asset file'], + ['resume', 'r', None, 'Resume at this index']] + +class SkelTest(object): + implements(IPlugin, ITest) + + shortName = "skeleton" + description = "Skeleton plugin" + requirements = None + arguments = SkelArgs + + def startTest(): + pass + +skel = SkelTest() + diff --git a/plugoo/interface.py b/plugoo/interface.py new file mode 100644 index 0000000..72f58d7 --- /dev/null +++ b/plugoo/interface.py @@ -0,0 +1,2 @@ +from zope.interface import implements, Interface, Attribute + diff --git a/plugoo/nodes.py b/plugoo/nodes.py index 5b16d22..6cdba65 100644 --- a/plugoo/nodes.py +++ b/plugoo/nodes.py @@ -33,6 +33,10 @@ class Node(object): self.address = address self.port = port +class LocalNode(object): + def __init__(self): + pass + """ []: node = NetworkNode("192.168.0.112", 5555, "SOCKS5") []: node_socket = node.wrap_socket() @@ -91,7 +95,7 @@ class CodeExecNode(Node): def get_status(self): pass -class PlanetLab(Node, CodeExecNode): +class PlanetLab(CodeExecNode): def __init__(self, address, auth_creds, ooni): self.auth_creds = auth_creds @@ -147,7 +151,7 @@ class PlanetLab(Node, CodeExecNode): except paramiko.SSHException: print 'Public key authentication to PlanetLab node %s failed.' % machinename, - def _get_command: + def _get_command(): pass def ssh_and_run_(slicename, machinename, command): @@ -169,8 +173,8 @@ class PlanetLab(Node, CodeExecNode): """Attempt to rsync a tree to the PL node.""" pass - def add_unit: + def add_unit(): pass - def get_status: + def get_status(): pass diff --git a/plugoo/tests.py b/plugoo/tests.py index 9a9f1de..4c3e1a1 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -1,10 +1,13 @@ import os from datetime import datetime import yaml +from zope.interface import Interface, Attribute import logging import itertools import gevent +from twisted.internet import reactor, defer +from twisted.python import failure from plugoo.reports import Report class Test: @@ -106,48 +109,11 @@ class Test: #print "JOB VAL: %s" % job.value self.logger.info("Writing report(s)") self.report(job.value) - job.kill() + job.kill() jobs = [] else: self.logger.error("No Assets! Dying!") -class WorkUnit(object): - """ - This is an object responsible for completing WorkUnits it will - return its result in a deferred. - - The execution of a unit of work should be Atomic. - - Reporting to the OONI-net happens on completion of a Unit of Work. - - @Node node: This represents the node associated with the Work Unit - @Asset asset: This is the asset associated with the Work Unit - @Test test: This represents the Test to be with the specified assets - @ivar arguments: These are the extra attributes to be passsed to the Test - """ - - node = None - asset = None - test = None - arguments = None - - def __init__(self, node, asset, test, arguments): - self.assetGenerator = asset() - self.Test = test - self.node = node - self.arguments = arguments - - def next(): - """ - Launches the Unit of Work with the specified assets on the node. - """ - try: - asset = self.assetGenerator.next() - yield self.Test(asset, self.node, self.arguments) - except StopIteration: - raise StopIteration - - class ITest(Interface): """ This interface represents an OONI test. It fires a deferred on completion. @@ -156,10 +122,10 @@ class ITest(Interface): shortName = Attribute("""A short user facing description for this test""") description = Attribute("""A string containing a longer description for the test""") - requirements = Attribtue("""What is required to run this this test, for example raw socket access or UDP or TCP""") + requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""") - deferred = Attribute("""This will be fired on test completion""") - node = Attribute("""This represents the node that will run the test""") + #deferred = Attribute("""This will be fired on test completion""") + #node = Attribute("""This represents the node that will run the test""") arguments = Attribute("""These are the arguments to be passed to the test for it's execution""") def startTest(): @@ -167,53 +133,8 @@ class ITest(Interface): Launches the Test with the specified arguments on a node. """ -class WorkGenerator(object): - """ - Factory responsible for creating units of work. - - This shall be run on the machine running OONI-cli. The returned WorkUnits - can either be run locally or on a remote OONI Node or Network Node. - """ - node = LocalNode - size = 10 - - def __init__(self, assets): - self.assets = assets() - - def next(self): - # Plank asset - p_asset = [] - for i in xrange(0, self.size): - p_asset.append(self.assets.next()) - yield WorkUnit(p_asset) - -def spawnDeferredTests(workunit, n): - def callback(result): - pass - - def errback(reason): - pass - - # XXX find more elegant solution to having 2 generators - workgenA = WorkGenerator(assets) - - for workunit in workgen: - deferredList = [] - workunitB = workunit - for i in range(n): - try: - test = workunit.next() - except StopIteration: - pass - - deferred = test.deferred - deferred.addCallback(callback).addErrback(errback) - - deferredList.append(deferred) - test.startTest() - - -class HTTPRequestTest(HTTPClient): +#class HTTPRequestTest(HTTPClient): +class HTTPRequestTest(object): """ This is an example of how I would like to be able to write a test. @@ -221,9 +142,9 @@ class HTTPRequestTest(HTTPClient): kind of API that I am attempting to achieve to simplify the writing of tests. - """ implements(ITest) + """ def startTest(): # The response object should also contain the request """ @@ -231,7 +152,6 @@ class HTTPRequestTest(HTTPClient): 'runtime': ..., 'timestamp': ...}, 'request': {'headers': ..., 'content', 'timestamp', ...} } - """ response = self.http_request(address, headers) if response.headers['content'].matches("Some string"): self.censorship = True @@ -240,9 +160,42 @@ class HTTPRequestTest(HTTPClient): self.censorship = False return response + """ + pass + +class TwistedTest(object): + def __init__(self, asset, node, arguments, ooninet=None): + self.asset = asset + self.node = node + self.arguments = arguments + self.start_time = datetime.now() + #self.ooninet = ooninet + + def __repr__(self): + return "<TwistedTest %s %s %s>" % (self.arguments, self.asset, self.node) + + def finished(self, result): + #self.ooninet.report(result) + self.end_time = datetime.now() + result['start_time'] = self.start_time + result['end_time'] = self.end_time + result['run_time'] = self.end_time - self.start_time + return self.d.callback(result) + + def startTest(self): + self.d = defer.Deferred() + result = {} + reactor.callLater(2.0, self.finished, result) + return self.d + + +class StupidTest(TwistedTest): + def __repr__(self): + return "<StupidTest %s %s %s>" % (self.arguments, self.asset, self.node) + class TwistedTestFactory(object): - test = TwistedTest + test = StupidTest def __init__(self, assets, node, idx=0): diff --git a/plugoo/work.py b/plugoo/work.py new file mode 100644 index 0000000..53cb18c --- /dev/null +++ b/plugoo/work.py @@ -0,0 +1,146 @@ +# -*- coding: UTF-8 +""" + work.py + ********** + + This contains all code related to generating + Units of Work and processing it. + + :copyright: (c) 2012 by Arturo Filastò. + :license: see LICENSE for more details. + +""" +from datetime import datetime +import yaml + +from zope.interface import Interface, Attribute + +from twisted.python import failure +from twisted.internet import reactor, defer + +from plugoo.nodes import LocalNode + +class Worker(object): + """ + This is the core of OONI. It takes as input Work Units and + runs them concurrently. + """ + def __init__(self, maxconcurrent=10): + self.maxconcurrent = maxconcurrent + self._running = 0 + self._queued = [] + + def _run(self, r): + self._running -= 1 + if self._running < self.maxconcurrent and self._queued: + workunit, d = self._queued.pop(0) + for work in workunit: + self._running += 1 + actuald = work.startTest().addBoth(self._run) + if isinstance(r, failure.Failure): + r.trap() + + print "Callback fired!" + print r['start_time'] + print r['end_time'] + print r['run_time'] + print repr(r) + return r + + def push(self, workunit): + if self._running < self.maxconcurrent: + for work in workunit: + self._running += 1 + work.startTest().addBoth(self._run) + return + d = defer.Deferred() + self._queued.append((workunit, d)) + return d + +class WorkUnit(object): + """ + This is an object responsible for completing WorkUnits it will + return its result in a deferred. + + The execution of a unit of work should be Atomic. + + Reporting to the OONI-net happens on completion of a Unit of Work. + + @Node node: This represents the node associated with the Work Unit + @Asset asset: This is the asset associated with the Work Unit + @Test test: This represents the Test to be with the specified assets + @ivar arguments: These are the extra attributes to be passsed to the Test + """ + + node = None + asset = None + test = None + arguments = None + + def __init__(self, node, asset, test, idx, arguments=None): + self.asset = asset + self.assetGenerator = iter(asset) + self.Test = test + self.node = node + self.arguments = arguments + self.idx = idx + + def __iter__(self): + return self + + def __repr__(self): + return "<WorkUnit %s %s %s>" % (self.arguments, self.Test, self.idx) + + def serialize(self): + """ + Serialize this unit of work for RPC activity. + """ + return yaml.dump(self) + + def next(self): + """ + Launches the Unit of Work with the specified assets on the node. + """ + try: + asset = self.assetGenerator.next() + return self.Test(asset, self.node, self.arguments) + except StopIteration: + raise StopIteration + + +class WorkGenerator(object): + """ + Factory responsible for creating units of work. + + This shall be run on the machine running OONI-cli. The returned WorkUnits + can either be run locally or on a remote OONI Node or Network Node. + """ + node = LocalNode + size = 10 + + def __init__(self, asset, test, arguments=None, start=None): + self.assetGenerator = asset() + self.Test = test + self.arguments = arguments + self.idx = 0 + if start: + self.skip(start) + + def __iter__(self): + return self + + def skip(self, start): + for j in xrange(0, start-1): + for i in xrange(0, self.size): + self.assetGenerator.next() + self.idx += 1 + + def next(self): + # Plank asset + p_asset = [] + for i in xrange(0, self.size): + p_asset.append(self.assetGenerator.next()) + self.idx += 1 + return WorkUnit(self.node, p_asset, self.Test, self.idx, self.arguments) + + diff --git a/plugoo/workers.py b/plugoo/workers.py deleted file mode 100644 index 3c8397d..0000000 --- a/plugoo/workers.py +++ /dev/null @@ -1,36 +0,0 @@ -import gevent -from gevent.pool import Pool - -class WorkFactory: - """ - This class is responsible for producing - units of work. - """ - def __init__(self, assets=None, - nodes=None, rule=None): - pass - - def _process_rule(self): - pass - - def get_work_unit(): - pass - -class UnitOfWork: - def __init__(self, tests, poolsize=20, - unit_of_work=None): - pass - - def _read_unit_of_work(self): - pass - - def _build_pools(self): - for i, x in enumerate(self.tests): - if i % self.poolsize == 0: - - - def do(self): - with gevent.Timeout(): - self.pool.join() - - diff --git a/skel.py b/skel.py new file mode 100644 index 0000000..8982106 --- /dev/null +++ b/skel.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 +""" + XXX + *** + + :copyright: (c) 2012 by Arturo Filastò + :license: see LICENSE for more details. +""" + diff --git a/unittest/tests.py b/unittest/tests.py new file mode 100644 index 0000000..da7c7d0 --- /dev/null +++ b/unittest/tests.py @@ -0,0 +1,27 @@ +from twisted.internet import reactor +from plugoo import * + +class StupidAsset(object): + def __init__(self): + self.idx = 0 + + def __iter__(self): + return self + + def next(self): + if self.idx > 30: + raise StopIteration + self.idx += 1 + return self.idx + +wgen = work.WorkGenerator(StupidAsset, tests.StupidTest, {'bla': 'aaa'}, start=0) +worker = work.Worker() +for x in wgen: + print "------" + print "Work unit" + print "------" + worker.push(x) + print "------" + +reactor.run() +

1 0

[ooni-probe/master] Merge branch 'twisted' of git://github.com/hellais/ooni-probe
by art＠torproject.org 30 May '12

30 May '12

commit abfcf7cd3d1c0a159ee29373ccd7e021b939ac6c Merge: 341af8e 507d0d1 Author: x <x(a)lulzmachine.(none)> Date: Wed May 23 13:09:22 2012 +0200 Merge branch 'twisted' of git://github.com/hellais/ooni-probe lib/traceroute.py | 32 ++++ lib/txtraceroute.py | 389 ++++++++++++++++++++++++++++++++++++++++++++++++++ oonicli.py | 142 ++++++++++++++++++ ooniprobe.py | 3 +- plugins/__init__.py | 3 + plugins/dropin.cache | 48 ++++++ plugins/skel.py | 20 +++ plugoo/interface.py | 2 + plugoo/nodes.py | 12 +- plugoo/reports.py | 5 +- plugoo/tests.py | 99 +++++++++++++ plugoo/work.py | 162 +++++++++++++++++++++ plugoo/workers.py | 36 ----- skel.py | 10 ++ unittest/tests.py | 27 ++++ 15 files changed, 946 insertions(+), 44 deletions(-)

1 0

[ooni-probe/master] Finish a working ooniclient implementation
by art＠torproject.org 30 May '12

30 May '12

commit 3a31b613c62706d8eb9617c143981a5b1b95df0c Author: Arturo Filastò <hellais(a)torproject.org> Date: Wed May 2 21:32:37 2012 +0200 Finish a working ooniclient implementation --- oonicli.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++---- plugins/skel.py | 16 ++++----- plugoo/tests.py | 10 +---- plugoo/work.py | 16 ++++++++- unittest/tests.py | 4 +- 5 files changed, 112 insertions(+), 29 deletions(-) diff --git a/oonicli.py b/oonicli.py index d77ad0f..0a9c1cc 100755 --- a/oonicli.py +++ b/oonicli.py @@ -8,18 +8,27 @@ # :license: see LICENSE for more details. # -from plugoo import tests +import sys +from plugoo import tests, work from twisted.python import usage - from twisted.plugin import getPlugins +from twisted.internet import reactor from zope.interface.exceptions import BrokenImplementation from zope.interface.exceptions import BrokenMethodImplementation from zope.interface.verify import verifyObject +from pprint import pprint +from logo import getlogo import plugins +__version__ = "0.0.1-prealpha" + def retrieve_plugoo(): + """ + Get all the plugins that implement the ITest interface and get the data + associated to them into a dict. + """ interface = tests.ITest d = {} error = False @@ -28,11 +37,11 @@ def retrieve_plugoo(): verifyObject(interface, p) d[p.shortName] = p except BrokenImplementation, bi: - print "Plugin Broke" + print "Plugin Broken" print bi error = True except BrokenMethodImplementation, bmi: - print "Plugin Broke" + print "Plugin Broken" error = True if error != False: print "Plugin Loaded!" @@ -40,18 +49,88 @@ def retrieve_plugoo(): plugoo = retrieve_plugoo() +class StupidAsset(object): + def __init__(self): + self.idx = 0 + + def __iter__(self): + return self + + def next(self): + if self.idx > 30: + raise StopIteration + self.idx += 1 + return self.idx + +def runTest(test, options): + pprint(test) + wgen = work.WorkGenerator(StupidAsset, plugoo[test].__class__, + dict(options), + start=options['resume']) + + worker = work.Worker() + for x in wgen: + print "------" + print "Work unit" + print "------" + print x.serialize() + worker.push(x) + print "------" + + reactor.run() + class Options(usage.Options): tests = plugoo.keys() subCommands = [] for test in tests: - subCommands.append([test, None, plugoo[test].arguments, "Run the %s test" % test]) + subCommands.append([test, None, plugoo[test].options, "Run the %s test" % test]) + + optFlags = [ + ['local', 'l', "If the test should be run locally (without having oonid running)"], + ['status', 'x', 'Show current state'], + ['restart', 'r', 'Restart OONI'] + ] optParameters = [ - ['status', 's', 0, 'Show current state'], - ['restart', 'r', None, 'Restart OONI'], - ['node', 'n', 'localhost:31415', 'Select target node'] + ['node', 'n', 'localhost:31415', 'Select target node'], + ['ooninet', 'o', 'localhost:4242', "Select OONI-net address for reporting"], + ['password', 'p', 'opennetwork', "Specify the password for authentication"], ] + def opt_version(self): + """ + Display OONI version and exit. + """ + print "OONI version:", __version__ + sys.exit(0) + + def __str__(self): + """ + Hack to get the sweet ascii art into the help output and replace the + strings "Commands" with "Tests". + """ + return getlogo() + '\n' + self.getSynopsis() + '\n' + \ + self.getUsage(width=None).replace("Commands:", "Tests:") + config = Options() config.parseOptions() +if config['status']: + print "oonid is not running." + sys.exit(0) + +if config['restart']: + print "Restarting oonid." + sys.exit(0) + +if not config.subCommand: + print "Error! No Test Specified." + config.opt_help() + sys.exit(1) + +if config['local']: + runTest(config.subCommand, config.subOptions) + +else: + print "The test will be run on the node %s" % config['node'] + diff --git a/plugins/skel.py b/plugins/skel.py index d27932a..41420b1 100644 --- a/plugins/skel.py +++ b/plugins/skel.py @@ -1,22 +1,20 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from plugoo.tests import ITest +from plugoo.tests import ITest, TwistedTest class SkelArgs(usage.Options): optParameters = [['assets', 'a', None, 'Asset file'], - ['resume', 'r', None, 'Resume at this index']] + ['resume', 'r', 0, 'Resume at this index']] -class SkelTest(object): +class SkelTest(TwistedTest): implements(IPlugin, ITest) shortName = "skeleton" description = "Skeleton plugin" requirements = None - arguments = SkelArgs - - def startTest(): - pass - -skel = SkelTest() + options = SkelArgs +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +skel = SkelTest(None, None) diff --git a/plugoo/tests.py b/plugoo/tests.py index 4c3e1a1..41363ee 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -126,7 +126,7 @@ class ITest(Interface): #deferred = Attribute("""This will be fired on test completion""") #node = Attribute("""This represents the node that will run the test""") - arguments = Attribute("""These are the arguments to be passed to the test for it's execution""") + options = Attribute("""These are the arguments to be passed to the test for it's execution""") def startTest(): """ @@ -164,9 +164,8 @@ class HTTPRequestTest(object): pass class TwistedTest(object): - def __init__(self, asset, node, arguments, ooninet=None): + def __init__(self, asset, arguments, ooninet=None): self.asset = asset - self.node = node self.arguments = arguments self.start_time = datetime.now() #self.ooninet = ooninet @@ -188,11 +187,6 @@ class TwistedTest(object): reactor.callLater(2.0, self.finished, result) return self.d - -class StupidTest(TwistedTest): - def __repr__(self): - return "<StupidTest %s %s %s>" % (self.arguments, self.asset, self.node) - class TwistedTestFactory(object): test = StupidTest diff --git a/plugoo/work.py b/plugoo/work.py index 53cb18c..809e9bd 100644 --- a/plugoo/work.py +++ b/plugoo/work.py @@ -31,10 +31,13 @@ class Worker(object): self._queued = [] def _run(self, r): + print "RUNNING" self._running -= 1 if self._running < self.maxconcurrent and self._queued: workunit, d = self._queued.pop(0) for work in workunit: + print "Going over workunits bis" + print work self._running += 1 actuald = work.startTest().addBoth(self._run) if isinstance(r, failure.Failure): @@ -48,8 +51,11 @@ class Worker(object): return r def push(self, workunit): + print "PUSHING" if self._running < self.maxconcurrent: for work in workunit: + print "Going over work units" + print dir(work) self._running += 1 work.startTest().addBoth(self._run) return @@ -102,9 +108,15 @@ class WorkUnit(object): Launches the Unit of Work with the specified assets on the node. """ try: - asset = self.assetGenerator.next() - return self.Test(asset, self.node, self.arguments) + asset = self.assetGenerator.next() + print "Next shit.." + print asset + ret = self.Test(asset, self.arguments) + print type(ret) + print repr(ret) + return ret except StopIteration: + print "Stopped iteration!" raise StopIteration diff --git a/unittest/tests.py b/unittest/tests.py index da7c7d0..10887b2 100644 --- a/unittest/tests.py +++ b/unittest/tests.py @@ -1,5 +1,5 @@ from twisted.internet import reactor -from plugoo import * +from plugoo import work, tests class StupidAsset(object): def __init__(self): @@ -14,7 +14,7 @@ class StupidAsset(object): self.idx += 1 return self.idx -wgen = work.WorkGenerator(StupidAsset, tests.StupidTest, {'bla': 'aaa'}, start=0) +wgen = work.WorkGenerator(StupidAsset, tests.StupidTest(None, None, None, None), {'bla': 'aaa'}, start=0) worker = work.Worker() for x in wgen: print "------"

1 0

[ooni-probe/master] Properly accept assets when running tests
by art＠torproject.org 30 May '12

30 May '12

commit 6306a4e3b5670cd68fd450cd9ed39f88c2b528ac Author: Arturo Filastò <hellais(a)torproject.org> Date: Wed May 2 22:14:37 2012 +0200 Properly accept assets when running tests --- oonicli.py | 20 ++++++++++---------- plugins/skel.py | 2 +- plugoo/tests.py | 6 ++++-- plugoo/work.py | 42 +++++++++++++++++++++++------------------- 4 files changed, 38 insertions(+), 32 deletions(-) diff --git a/oonicli.py b/oonicli.py index 0a9c1cc..c2d01d2 100755 --- a/oonicli.py +++ b/oonicli.py @@ -9,7 +9,7 @@ # import sys -from plugoo import tests, work +from plugoo import tests, work, assets from twisted.python import usage from twisted.plugin import getPlugins @@ -62,20 +62,20 @@ class StupidAsset(object): self.idx += 1 return self.idx + def runTest(test, options): - pprint(test) - wgen = work.WorkGenerator(StupidAsset, plugoo[test].__class__, - dict(options), - start=options['resume']) + asset = None + if options['asset']: + print options['asset'] + asset = assets.Asset(options['asset']) + print asset + + wgen = work.WorkGenerator(asset, plugoo[test].__class__, + dict(options), start=options['resume']) worker = work.Worker() for x in wgen: - print "------" - print "Work unit" - print "------" - print x.serialize() worker.push(x) - print "------" reactor.run() diff --git a/plugins/skel.py b/plugins/skel.py index 41420b1..0427239 100644 --- a/plugins/skel.py +++ b/plugins/skel.py @@ -4,7 +4,7 @@ from twisted.plugin import IPlugin from plugoo.tests import ITest, TwistedTest class SkelArgs(usage.Options): - optParameters = [['assets', 'a', None, 'Asset file'], + optParameters = [['asset', 'a', None, 'Asset file'], ['resume', 'r', 0, 'Resume at this index']] class SkelTest(TwistedTest): diff --git a/plugoo/tests.py b/plugoo/tests.py index 41363ee..e74586c 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -171,10 +171,11 @@ class TwistedTest(object): #self.ooninet = ooninet def __repr__(self): - return "<TwistedTest %s %s %s>" % (self.arguments, self.asset, self.node) + return "<TwistedTest %s %s>" % (self.arguments, self.asset) def finished(self, result): #self.ooninet.report(result) + print "FINIHSED" self.end_time = datetime.now() result['start_time'] = self.start_time result['end_time'] = self.end_time @@ -182,6 +183,7 @@ class TwistedTest(object): return self.d.callback(result) def startTest(self): + print "Starting test" self.d = defer.Deferred() result = {} reactor.callLater(2.0, self.finished, result) @@ -189,7 +191,7 @@ class TwistedTest(object): class TwistedTestFactory(object): - test = StupidTest + test = None def __init__(self, assets, node, idx=0): diff --git a/plugoo/work.py b/plugoo/work.py index 809e9bd..7be4758 100644 --- a/plugoo/work.py +++ b/plugoo/work.py @@ -31,13 +31,10 @@ class Worker(object): self._queued = [] def _run(self, r): - print "RUNNING" self._running -= 1 if self._running < self.maxconcurrent and self._queued: workunit, d = self._queued.pop(0) for work in workunit: - print "Going over workunits bis" - print work self._running += 1 actuald = work.startTest().addBoth(self._run) if isinstance(r, failure.Failure): @@ -51,11 +48,8 @@ class Worker(object): return r def push(self, workunit): - print "PUSHING" if self._running < self.maxconcurrent: for work in workunit: - print "Going over work units" - print dir(work) self._running += 1 work.startTest().addBoth(self._run) return @@ -83,11 +77,13 @@ class WorkUnit(object): test = None arguments = None - def __init__(self, node, asset, test, idx, arguments=None): + def __init__(self, asset, test, idx, arguments=None): self.asset = asset - self.assetGenerator = iter(asset) + if not asset: + self.assetGenerator = iter([1]) + else: + self.assetGenerator = iter(asset) self.Test = test - self.node = node self.arguments = arguments self.idx = idx @@ -109,14 +105,9 @@ class WorkUnit(object): """ try: asset = self.assetGenerator.next() - print "Next shit.." - print asset ret = self.Test(asset, self.arguments) - print type(ret) - print repr(ret) return ret except StopIteration: - print "Stopped iteration!" raise StopIteration @@ -127,14 +118,14 @@ class WorkGenerator(object): This shall be run on the machine running OONI-cli. The returned WorkUnits can either be run locally or on a remote OONI Node or Network Node. """ - node = LocalNode size = 10 def __init__(self, asset, test, arguments=None, start=None): - self.assetGenerator = asset() + self.assetGenerator = asset self.Test = test self.arguments = arguments self.idx = 0 + self.end = False if start: self.skip(start) @@ -148,11 +139,24 @@ class WorkGenerator(object): self.idx += 1 def next(self): + if self.end: + raise StopIteration + + if not self.assetGenerator: + self.end = True + return WorkUnit(None, self.Test, self.idx, self.arguments) + # Plank asset p_asset = [] for i in xrange(0, self.size): - p_asset.append(self.assetGenerator.next()) - self.idx += 1 - return WorkUnit(self.node, p_asset, self.Test, self.idx, self.arguments) + try: + asset = self.assetGenerator.next() + p_asset.append(asset) + print asset + except StopIteration: + self.end = True + break + self.idx += 1 + return WorkUnit(p_asset, self.Test, self.idx, self.arguments)

1 0

[ooni-probe/master] Add ICMP traceroute
by art＠torproject.org 30 May '12

30 May '12

commit e452b755b750781b14b372780966e67a7af2f0fc Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 3 00:29:59 2012 +0200 Add ICMP traceroute --- lib/traceroute.py | 32 +++++ lib/txtraceroute.py | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++ oonicli.py | 16 ++- 3 files changed, 357 insertions(+), 5 deletions(-) diff --git a/lib/traceroute.py b/lib/traceroute.py new file mode 100644 index 0000000..98015ea --- /dev/null +++ b/lib/traceroute.py @@ -0,0 +1,32 @@ +import sys +import socket +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet import threads + +from txtraceroute import traceroute + +def run(target, src_port, dst_port): + res = [] + @defer.inlineCallbacks + def start_trace(target, **settings): + hops = yield traceroute(target, **settings) + for hop in hops: + res.append(hop.get()) + reactor.stop() + + settings = dict(hop_callback=None, + timeout=2, + max_tries=3, + max_hops=30) + try: + target = socket.gethostbyname(target) + except Exception, e: + print("could not resolve '%s': %s" % (target, str(e))) + sys.exit(1) + + reactor.callWhenRunning(start_trace, target, **settings) + reactor.run() + return res + +print run("google.com") diff --git a/lib/txtraceroute.py b/lib/txtraceroute.py new file mode 100644 index 0000000..f74dfb1 --- /dev/null +++ b/lib/txtraceroute.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python +# coding: utf-8 +# +# Copyright (c) 2012 Alexandre Fiori +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import json +import operator +import os +import socket +import struct +import sys +import time + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet import threads +from twisted.python import usage +from twisted.web.client import getPage + + +class iphdr(object): + def __init__(self, proto=socket.IPPROTO_ICMP, src="0.0.0.0", dst=None): + self.version = 4 + self.hlen = 5 + self.tos = 0 + self.length = 20 + self.id = os.getpid() + self.frag = 0 + self.ttl = 255 + self.proto = proto + self.cksum = 0 + self.src = src + self.saddr = socket.inet_aton(src) + self.dst = dst or "0.0.0.0" + self.daddr = socket.inet_aton(self.dst) + self.data = "" + + def assemble(self): + header = struct.pack('BBHHHBB', + (self.version & 0x0f) << 4 | (self.hlen & 0x0f), + self.tos, self.length + len(self.data), + socket.htons(self.id), self.frag, + self.ttl, self.proto) + return header + "\000\000" + self.saddr + self.daddr + self.data + + @classmethod + def disassemble(self, data): + ip = iphdr() + pkt = struct.unpack('!BBHHHBBH', data[:12]) + ip.version = (pkt[0] >> 4 & 0x0f) + ip.hlen = (pkt[0] & 0x0f) + ip.tos, ip.length, ip.id, ip.frag, ip.ttl, ip.proto, ip.cksum = pkt[1:] + ip.saddr = data[12:16] + ip.daddr = data[16:20] + ip.src = socket.inet_ntoa(ip.saddr) + ip.dst = socket.inet_ntoa(ip.daddr) + return ip + + def __repr__(self): + return "IP (tos %s, ttl %s, id %s, frag %s, proto %s, length %s) " \ + "%s -> %s" % \ + (self.tos, self.ttl, self.id, self.frag, self.proto, + self.length, self.src, self.dst) + +class icmphdr(object): + def __init__(self, data=""): + self.type = 8 + self.code = 0 + self.cksum = 0 + self.id = os.getpid() + self.sequence = 0 + self.data = data + + def assemble(self): + part1 = struct.pack("BB", self.type, self.code) + part2 = struct.pack("!HH", self.id, self.sequence) + cksum = self.checksum(part1 + "\000\000" + part2 + self.data) + cksum = struct.pack("!H", cksum) + return part1 + cksum + part2 + self.data + + @classmethod + def checksum(self, data): + if len(data) & 1: + data += "\0" + cksum = reduce(operator.add, + struct.unpack('!%dH' % (len(data) >> 1), data)) + cksum = (cksum >> 16) + (cksum & 0xffff) + cksum += (cksum >> 16) + cksum = (cksum & 0xffff) ^ 0xffff + return cksum + + @classmethod + def disassemble(self, data): + icmp = icmphdr() + pkt = struct.unpack("!BBHHH", data) + icmp.type, icmp.code, icmp.cksum, icmp.id, icmp.sequence = pkt + return icmp + + def __repr__(self): + return "ICMP (type %s, code %s, id %s, sequence %s)" % \ + (self.type, self.code, self.id, self.sequence) + + +(a)defer.inlineCallbacks +def geoip_lookup(ip): + try: + r = yield getPage("http://freegeoip.net/json/%s" % ip) + d = json.loads(r) + items = [d["country_name"], d["region_name"], d["city"]] + text = ", ".join([s for s in items if s]) + defer.returnValue(text.encode("utf-8")) + except Exception: + defer.returnValue("Unknown location") + + +(a)defer.inlineCallbacks +def reverse_lookup(ip): + try: + r = yield threads.deferToThread(socket.gethostbyaddr, ip) + defer.returnValue(r[0]) + except Exception: + defer.returnValue(None) + + +class Hop(object): + def __init__(self, target, ttl): + self.found = False + self.tries = 0 + self.last_try = 0 + self.remote_ip = None + self.remote_icmp = None + self.remote_host = None + self.location = "" + + self.ttl = ttl + self.ip = iphdr(dst=target) + self.ip.ttl = ttl + self.ip.id += ttl + + self.icmp = icmphdr("traceroute") + self.icmp.id = self.ip.id + self.ip.data = self.icmp.assemble() + + self._pkt = self.ip.assemble() + + @property + def pkt(self): + self.tries += 1 + self.last_try = time.time() + return self._pkt + + def get(self): + if self.found: + if self.remote_host: + ip = self.remote_host + else: + ip = self.remote_ip.src + ping = self.found - self.last_try + else: + ip = None + ping = None + + location = self.location if self.location else None + return {'ttl': self.ttl, 'ping': ping, 'ip': ip, 'location': location} + + def __repr__(self): + if self.found: + if self.remote_host: + ip = ":: %s" % self.remote_host + else: + ip = ":: %s" % self.remote_ip.src + ping = "%0.3fs" % (self.found - self.last_try) + else: + ip = "??" + ping = "-" + + location = ":: %s" % self.location if self.location else "" + return "%02d. %s %s %s" % (self.ttl, ping, ip, location) + + +class TracerouteProtocol(object): + def __init__(self, target, **settings): + self.target = target + self.settings = settings + self.fd = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_ICMP) + self.fd.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1) + + self.hops = [] + self.out_queue = [] + self.waiting = True + self.deferred = defer.Deferred() + + reactor.addReader(self) + reactor.addWriter(self) + + # send 1st probe packet + self.out_queue.append(Hop(self.target, 1)) + + def logPrefix(self): + return "TracerouteProtocol(%s)" % self.target + + def fileno(self): + return self.fd.fileno() + + @defer.inlineCallbacks + def hopFound(self, hop, ip, icmp): + hop.remote_ip = ip + hop.remote_icmp = icmp + + if (ip and icmp): + hop.found = time.time() + if self.settings.get("geoip_lookup") is True: + hop.location = yield geoip_lookup(ip.src) + + if self.settings.get("reverse_lookup") is True: + hop.remote_host = yield reverse_lookup(ip.src) + + ttl = hop.ttl + 1 + last = self.hops[-2:] + if len(last) == 2 and last[0].remote_ip == ip or \ + (ttl > (self.settings.get("max_hops", 30) + 1)): + done = True + else: + done = False + + if not done: + cb = self.settings.get("hop_callback") + if callable(cb): + yield defer.maybeDeferred(cb, hop) + + if not self.waiting: + if self.deferred: + self.deferred.callback(self.hops) + self.deferred = None + else: + self.out_queue.append(Hop(self.target, ttl)) + + def doRead(self): + if not self.waiting or not self.hops: + return + + pkt = self.fd.recv(4096) + + # disassemble ip header + ip = iphdr.disassemble(pkt[:20]) + if ip.proto != socket.IPPROTO_ICMP: + return + + found = False + + # disassemble icmp header + icmp = icmphdr.disassemble(pkt[20:28]) + if icmp.type == 0 and icmp.id == self.hops[-1].icmp.id: + found = True + elif icmp.type == 11: + # disassemble referenced ip header + ref = iphdr.disassemble(pkt[28:48]) + if ref.dst == self.target: + found = True + + if ip.src == self.target: + self.waiting = False + + if found: + self.hopFound(self.hops[-1], ip, icmp) + + def hopTimeout(self, *ign): + hop = self.hops[-1] + if not hop.found: + if hop.tries < self.settings.get("max_tries", 3): + # retry + self.out_queue.append(hop) + else: + # give up and move forward + self.hopFound(hop, None, None) + + def doWrite(self): + if self.waiting and self.out_queue: + hop = self.out_queue.pop(0) + pkt = hop.pkt + if not self.hops or (self.hops and hop.ttl != self.hops[-1].ttl): + self.hops.append(hop) + self.fd.sendto(pkt, (hop.ip.dst, 0)) + + timeout = self.settings.get("timeout", 1) + reactor.callLater(timeout, self.hopTimeout) + + def connectionLost(self, why): + pass + + +def traceroute(target, **settings): + tr = TracerouteProtocol(target, **settings) + return tr.deferred diff --git a/oonicli.py b/oonicli.py index c2d01d2..68384f8 100755 --- a/oonicli.py +++ b/oonicli.py @@ -63,7 +63,7 @@ class StupidAsset(object): return self.idx -def runTest(test, options): +def runTest(test, options, global_options): asset = None if options['asset']: print options['asset'] @@ -73,7 +73,10 @@ def runTest(test, options): wgen = work.WorkGenerator(asset, plugoo[test].__class__, dict(options), start=options['resume']) - worker = work.Worker() + if global_options['parallelism']: + wgen.size = int(global_options['parallelism']) + worker = work.Worker(wgen.size) + for x in wgen: worker.push(x) @@ -92,7 +95,8 @@ class Options(usage.Options): ] optParameters = [ - ['node', 'n', 'localhost:31415', 'Select target node'], + ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], + ['target-node', 't', 'localhost:31415', 'Select target node'], ['ooninet', 'o', 'localhost:4242', "Select OONI-net address for reporting"], ['password', 'p', 'opennetwork', "Specify the password for authentication"], ] @@ -129,8 +133,10 @@ if not config.subCommand: sys.exit(1) if config['local']: - runTest(config.subCommand, config.subOptions) + runTest(config.subCommand, config.subOptions, config) else: - print "The test will be run on the node %s" % config['node'] + print "This feature is currently not supported. :(" + print "Use -l to run the test locally." + sys.exit(0)

1 0

[ooni-probe/master] Implement UDP and TCP traceroute
by art＠torproject.org 30 May '12

30 May '12

commit 507d0d141092de53a5a5b9a1a75cb21a421f403d Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 3 02:00:29 2012 +0200 Implement UDP and TCP traceroute --- lib/traceroute.py | 4 +- lib/txtraceroute.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/lib/traceroute.py b/lib/traceroute.py index 98015ea..c02722a 100644 --- a/lib/traceroute.py +++ b/lib/traceroute.py @@ -18,7 +18,7 @@ def run(target, src_port, dst_port): settings = dict(hop_callback=None, timeout=2, max_tries=3, - max_hops=30) + max_hops=30, proto="tcp") try: target = socket.gethostbyname(target) except Exception, e: @@ -29,4 +29,4 @@ def run(target, src_port, dst_port): reactor.run() return res -print run("google.com") +print run("8.8.8.8", 80, 80) diff --git a/lib/txtraceroute.py b/lib/txtraceroute.py index f74dfb1..e18b558 100644 --- a/lib/txtraceroute.py +++ b/lib/txtraceroute.py @@ -81,6 +81,71 @@ class iphdr(object): (self.tos, self.ttl, self.id, self.frag, self.proto, self.length, self.src, self.dst) +class tcphdr(object): + def __init__(self, data="", dport=4242, sport=4242): + self.seq = 123132 + self.hlen = 44 + self.flags = 2 + self.wsize = 200 + self.cksum = 123 + self.options = 0 + self.mss = 1460 + self.dport = dport + self.sport = sport + + def assemble(self): + header = struct.pack("!HHL", self.sport, self.dport, self.seq) + header += '\00\00\00\00' + header += struct.pack("!HHH", (self.hlen & 0xff) << 10 | (self.flags & + 0xff), self.wsize, self.cksum) + # XXX There is something wrong here fixme + options = struct.pack("!LBBBBBB", self.mss, 1, 3, 3, 1, 1, 1) + options += struct.pack("!BBL", 8, 10, 1209452188) + options += '\00'*4 + options += struct.pack("!BB", 4, 2) + options += '\00' + return header+options + + @classmethod + def checksum(self, data): + pass + + def disassemble(self, data): + tcp = tcphdr() + pkt = struct.unpack("!HHLH", data[:20]) + tcp.sport, tcp.dport, tcp.seq = pkt[:3] + tcp.hlen = (pkt[4] >> 10 ) & 0xff + tcp.flags = pkf[4] & 0xff + tcp.wsize, tcp.cksum = struct.unpack("!HH", data[20:28]) + return tcp + +class udphdr(object): + def __init__(self, data="", dport=4242, sport=4242): + self.dport = dport + self.sport = sport + self.cksum = 0 + self.length = 0 + self.data = data + + def assemble(self): + self.length = len(self.data) + 8 + part1 = struct.pack("!HHH", self.sport, self.dport, self.length) + cksum = self.checksum(self.data) + cksum = struct.pack("!H", cksum) + return part1 + cksum + self.data + + @classmethod + def checksum(self, data): + # XXX implement proper checksum + cksum = 0 + return cksum + + def disassemble(self, data): + udp = udphdr() + pkt = struct.unpack("!HHHH", data) + udp.src_port, udp.dst_port, udp.length, udp.cksum = pkt + return udp + class icmphdr(object): def __init__(self, data=""): self.type = 8 @@ -142,7 +207,8 @@ def reverse_lookup(ip): class Hop(object): - def __init__(self, target, ttl): + def __init__(self, target, ttl, proto="icmp"): + self.proto = proto self.found = False self.tries = 0 self.last_try = 0 @@ -156,9 +222,18 @@ class Hop(object): self.ip.ttl = ttl self.ip.id += ttl - self.icmp = icmphdr("traceroute") - self.icmp.id = self.ip.id - self.ip.data = self.icmp.assemble() + if proto is "icmp": + self.icmp = icmphdr("traceroute") + self.icmp.id = self.ip.id + self.ip.data = self.icmp.assemble() + elif proto is "udp": + self.udp = udphdr("blabla") + self.ip.data = self.udp.assemble() + self.ip.proto = socket.IPPROTO_UDP + elif proto is "tcp": + self.tcp = tcphdr() + self.ip.data = self.tcp.assemble() + self.ip.proto = socket.IPPROTO_TCP self._pkt = self.ip.assemble() @@ -214,7 +289,7 @@ class TracerouteProtocol(object): reactor.addWriter(self) # send 1st probe packet - self.out_queue.append(Hop(self.target, 1)) + self.out_queue.append(Hop(self.target, 1, settings.get("proto"))) def logPrefix(self): return "TracerouteProtocol(%s)" % self.target @@ -253,7 +328,7 @@ class TracerouteProtocol(object): self.deferred.callback(self.hops) self.deferred = None else: - self.out_queue.append(Hop(self.target, ttl)) + self.out_queue.append(Hop(self.target, ttl, self.settings.get("proto"))) def doRead(self): if not self.waiting or not self.hops:

1 0